0: ch1=8074 mean# 8074 dev# 0 part# 0
1: ch1=8074 ch2=8074 mean# 8074 dev# 0 part# 0
2: ch1=13338 ch2=10884 mean# 12111 dev# 1227 part# 0.1013128561
3: ch1=18148 ch2=14148 mean# 16148 dev# 2000 part# 0.1238543473
4: ch1=20188 ch2=20182 mean# 20185 dev# 3 part# 0.0001486252167
5: ch1=27764 ch2=20680 mean# 24222 dev# 3542 part# 0.1462306994
6: ch1=28222 ch2=28296 mean# 28259 dev# 37 part# 0.001309317386
7: ch1=34120 ch2=30472 mean# 32296 dev# 1824 part# 0.05647758236
8: ch1=37842 ch2=34824 mean# 36333 dev# 1509 part# 0.04153249112
9: ch1=40380 ch2=40360 mean# 40370 dev# 10 part# 0.0002477086946
10: ch1=47458 ch2=41356 mean# 44407 dev# 3051 part# 0.06870538429
11: ch1=49004 ch2=47884 mean# 48444 dev# 560 part# 0.01155973908
12: ch1=53814 ch2=51148 mean# 52481 dev# 1333 part# 0.02539966845
13: ch1=58624 ch2=54412 mean# 56518 dev# 2106 part# 0.03726246506
14: ch1=60572 ch2=60538 mean# 60555 dev# 17 part# 0.0002807365205
15: ch1=68240 ch2=60944 mean# 64592 dev# 3648 part# 0.05647758236
16: ch1=68698 ch2=68560 mean# 68629 dev# 69 part# 0.001005405878
17: ch1=74596 ch2=70736 mean# 72666 dev# 1930 part# 0.0265598767
18: ch1=78318 ch2=75088 mean# 76703 dev# 1615 part# 0.02105523904
19: ch1=80764 ch2=80716 mean# 80740 dev# 24 part# 0.0002972504335
20: ch1=87934 ch2=81620 mean# 84777 dev# 3157 part# 0.03723887375
21: ch1=89480 ch2=88148 mean# 88814 dev# 666 part# 0.007498817754
22: ch1=94290 ch2=91412 mean# 92851 dev# 1439 part# 0.01549794833
23: ch1=99100 ch2=94676 mean# 96888 dev# 2212 part# 0.02283048468
24: ch1=100956 ch2=100894 mean# 100925 dev# 31 part# 0.0003071587813
25: ch1=108716 ch2=101208 mean# 104962 dev# 3754 part# 0.03576532459
26: ch1=109174 ch2=108824 mean# 108999 dev# 175 part# 0.001605519317
27: ch1=113984 ch2=112088 mean# 113036 dev# 948 part# 0.008386708659
28: ch1=119882 ch2=114264 mean# 117073 dev# 2809 part# 0.02399357666
29: ch1=121148 ch2=121072 mean# 121110 dev# 38 part# 0.0003137643465
30: ch1=128410 ch2=121884 mean# 125147 dev# 3263 part# 0.02607333775
31: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
32: ch1=126692 ch2=123602 mean# 125147 dev# 1545 part# 0.01234548171
33: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
34: ch1=123192 ch2=127102 mean# 125147 dev# 1955 part# 0.01562162896
35: ch1=129004 ch2=121290 mean# 125147 dev# 3857 part# 0.03081975597
36: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
37: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
38: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
39: ch1=123686 ch2=126608 mean# 125147 dev# 1461 part# 0.01167427106
40: ch1=128502 ch2=121792 mean# 125147 dev# 3355 part# 0.02680847324
41: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
42: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
43: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
44: ch1=123096 ch2=127198 mean# 125147 dev# 2051 part# 0.01638872686
45: ch1=129092 ch2=121202 mean# 125147 dev# 3945 part# 0.03152292904
46: ch1=121424 ch2=128870 mean# 125147 dev# 3723 part# 0.02974901516
47: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
48: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
49: ch1=123590 ch2=126704 mean# 125147 dev# 1557 part# 0.01244136895
50: ch1=128590 ch2=121704 mean# 125147 dev# 3443 part# 0.0275116463
51: ch1=121878 ch2=128416 mean# 125147 dev# 3269 part# 0.02612128137
52: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
53: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
54: ch1=123000 ch2=127244 mean# 125122 dev# 2122 part# 0.01695944758
55: ch1=128046 ch2=122198 mean# 125122 dev# 2924 part# 0.02336919167
56: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
57: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
58: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
59: ch1=122410 ch2=127834 mean# 125122 dev# 2712 part# 0.02167484535
60: ch1=128636 ch2=121608 mean# 125122 dev# 3514 part# 0.02808458944
61: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
62: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
63: ch1=125096 ch2=125148 mean# 125122 dev# 26 part# 0.0002077971899
64: ch1=122908 ch2=127336 mean# 125122 dev# 2214 part# 0.01769472994
65: ch1=128138 ch2=122106 mean# 125122 dev# 3016 part# 0.02410447403
66: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
67: ch1=126642 ch2=123602 mean# 125122 dev# 1520 part# 0.01214814341
68: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
69: ch1=122322 ch2=127922 mean# 125122 dev# 2800 part# 0.02237815892
70: ch1=128732 ch2=121512 mean# 125122 dev# 3610 part# 0.0288518406
71: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
72: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
73: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
74: ch1=123908 ch2=126336 mean# 125122 dev# 1214 part# 0.00970253033
75: ch1=128234 ch2=122010 mean# 125122 dev# 3112 part# 0.0248717252
76: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
77: ch1=126646 ch2=123598 mean# 125122 dev# 1524 part# 0.01218011221
78: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
79: ch1=123322 ch2=126922 mean# 125122 dev# 1800 part# 0.0143859593
80: ch1=128828 ch2=121416 mean# 125122 dev# 3706 part# 0.02961909177
81: ch1=121840 ch2=128404 mean# 125122 dev# 3282 part# 0.02623039913
82: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
83: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
84: ch1=123820 ch2=126424 mean# 125122 dev# 1302 part# 0.0104058439
85: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
86: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
87: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
88: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
89: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
90: ch1=128920 ch2=121374 mean# 125147 dev# 3773 part# 0.03014854531
91: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
92: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
93: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
94: ch1=123770 ch2=126524 mean# 125147 dev# 1377 part# 0.0110030604
95: ch1=128418 ch2=121876 mean# 125147 dev# 3271 part# 0.02613726258
96: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
97: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
98: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
99: ch1=123180 ch2=127114 mean# 125147 dev# 1967 part# 0.0157175162
100: ch1=126832 ch2=122374 ch3=1088 mean# 83431.33333 dev# 58253.9661 part# 0.6982264788
101: ch1=119248 ch2=125606 ch3=5440 mean# 83431.33333 dev# 55209.25102 part# 0.6617328145
102: ch1=123198 ch2=119712 ch3=7384 mean# 83431.33333 dev# 53792.41415 part# 0.6447507429
103: ch1=118610 ch2=120800 ch3=10884 mean# 83431.33333 dev# 51306.5019 part# 0.6149548359
104: ch1=120786 ch2=118624 ch3=10884 mean# 83431.33333 dev# 51306.30397 part# 0.6149524636
105: ch1=119344 ch2=115714 ch3=15236 mean# 83431.33333 dev# 48244.14881 part# 0.578249764
106: ch1=112258 ch2=118624 ch3=19412 mean# 83431.33333 dev# 45343.04628 part# 0.5434774259
107: ch1=115342 ch2=114272 ch3=20680 mean# 83431.33333 dev# 44374.04347 part# 0.5318630507
108: ch1=112078 ch2=114272 ch3=23944 mean# 83431.33333 dev# 42073.43207 part# 0.504288142
109: ch1=112988 ch2=112096 ch3=25210 mean# 83431.33333 dev# 41170.31015 part# 0.4934634089
110: ch1=111048 ch2=109858 ch3=29388 mean# 83431.33333 dev# 38217.49541 part# 0.4580712531
111: ch1=107722 ch2=111008 ch3=31564 mean# 83431.33333 dev# 36700.26929 part# 0.4398859257
112: ch1=107264 ch2=107744 ch3=35286 mean# 83431.33333 dev# 34044.45566 part# 0.4080535968
113: ch1=108196 ch2=105568 ch3=36530 mean# 83431.33333 dev# 33181.6003 part# 0.3977114948
114: ch1=102298 ch2=107660 ch3=40336 mean# 83431.33333 dev# 30551.52563 part# 0.3661876708
115: ch1=106818 ch2=102108 ch3=41368 mean# 83431.33333 dev# 29805.35783 part# 0.3572441748
116: ch1=99640 ch2=104934 ch3=45720 mean# 83431.33333 dev# 26753.38116 part# 0.3206634737
117: ch1=103404 ch2=99078 ch3=47812 mean# 83431.33333 dev# 25248.5148 part# 0.3026262891
118: ch1=97914 ch2=101220 ch3=51160 mean# 83431.33333 dev# 22859.15754 part# 0.273987681
119: ch1=100090 ch2=97956 ch3=52248 mean# 83431.33333 dev# 22067.15051 part# 0.2644947603
120: ch1=98242 ch2=96082 ch3=55970 mean# 83431.33333 dev# 19438.10727 part# 0.2329832989
121: ch1=92474 ch2=99040 ch3=58780 mean# 83431.33333 dev# 17636.02875 part# 0.2113837578
122: ch1=96196 ch2=93142 ch3=60956 mean# 83431.33333 dev# 15941.29199 part# 0.1910708046
123: ch1=92478 ch2=93596 ch3=64220 mean# 83431.33333 dev# 13592.12952 part# 0.1629139674
124: ch1=91390 ch2=92508 ch3=66396 mean# 83431.33333 dev# 12054.44364 part# 0.1444834112
125: ch1=90132 ch2=90044 ch3=70118 mean# 83431.33333 dev# 9414.016831 part# 0.1128355074
126: ch1=87038 ch2=91416 ch3=71840 mean# 83431.33333 dev# 8388.920365 part# 0.100548799
127: ch1=88126 ch2=87064 ch3=75104 mean# 83431.33333 dev# 5904.253909 part# 0.07076782395
128: ch1=87330 ch2=85976 ch3=76988 mean# 83431.33333 dev# 4589.534278 part# 0.05500971991
129: ch1=83608 ch2=85976 ch3=80710 mean# 83431.33333 dev# 2153.461916 part# 0.02581118903
130: ch1=86820 ch2=81834 ch3=81640 mean# 83431.33333 dev# 2397.457728 part# 0.02873569955
131: ch1=81136 ch2=84254 ch3=84904 mean# 83431.33333 dev# 1644.595459 part# 0.01971196423
132: ch1=85172 ch2=82394 ch3=82728 mean# 83431.33333 dev# 1238.367025 part# 0.01484294899
133: ch1=81358 ch2=84888 ch3=84048 mean# 83431.33333 dev# 1505.641244 part# 0.01804647228
134: ch1=87034 ch2=81624 ch3=81636 mean# 83431.33333 dev# 2547.474741 part# 0.03053378915
135: ch1=78960 ch2=85346 ch3=85988 mean# 83431.33333 dev# 3172.554947 part# 0.03802594085
136: ch1=82564 ch2=83918 ch3=83812 mean# 83431.33333 dev# 614.8221062 part# 0.007369199096
137: ch1=84678 ch2=82716 ch3=82900 mean# 83431.33333 dev# 884.7211739 part# 0.01060418357
138: ch1=84858 ch2=82716 ch3=82720 mean# 83431.33333 dev# 1008.806996 part# 0.01209146439
139: ch1=83312 ch2=83174 ch3=83808 mean# 83431.33333 dev# 272.236825 part# 0.003263004607
140: ch1=80764 ch2=84812 ch3=84718 mean# 83431.33333 dev# 1886.479849 part# 0.02261116746
141: ch1=84402 ch2=83176 ch3=82716 mean# 83431.33333 dev# 711.5922678 part# 0.008529077019
142: ch1=81598 ch2=84892 ch3=83804 mean# 83431.33333 dev# 1370.345293 part# 0.01642482792
143: ch1=86408 ch2=81628 ch3=82258 mean# 83431.33333 dev# 2120.476886 part# 0.02541583361
144: ch1=80666 ch2=84892 ch3=84736 mean# 83431.33333 dev# 1956.422813 part# 0.02344949714
145: ch1=87018 ch2=81712 ch3=81564 mean# 83431.33333 dev# 2536.875944 part# 0.03040675299
146: ch1=81148 ch2=84262 ch3=84884 mean# 83431.33333 dev# 1634.406994 part# 0.01958984627
147: ch1=84236 ch2=83350 ch3=82708 mean# 83431.33333 dev# 626.4488983 part# 0.007508556717
148: ch1=81560 ch2=84854 ch3=83880 mean# 83431.33333 dev# 1381.686248 part# 0.01656075953
149: ch1=87050 ch2=81624 ch3=81620 mean# 83431.33333 dev# 2558.78426 part# 0.03066934397
150: ch1=80064 ch2=85346 ch3=84884 mean# 83431.33333 dev# 2388.522742 part# 0.02862860566
151: ch1=83478 ch2=83478 ch3=83338 mean# 83431.33333 dev# 65.99663291 part# 0.0007910293444
152: ch1=83782 ch2=82720 ch3=83792 mean# 83431.33333 dev# 503.005191 part# 0.006028972221
153: ch1=82694 ch2=83808 ch3=83792 mean# 83431.33333 dev# 521.414316 part# 0.006249622236
154: ch1=84870 ch2=82720 ch3=82704 mean# 83431.33333 dev# 1017.311926 part# 0.01219340367
155: ch1=80518 ch2=84896 ch3=84880 mean# 83431.33333 dev# 2060.048112 part# 0.02469154009
156: ch1=85328 ch2=82720 ch3=82246 mean# 83431.33333 dev# 1355.034399 part# 0.016241313
157: ch1=82698 ch2=83808 ch3=83788 mean# 83431.33333 dev# 518.6092513 part# 0.006216000999
158: ch1=84874 ch2=82720 ch3=82700 mean# 83431.33333 dev# 1020.152058 part# 0.01222744523
159: ch1=80780 ch2=84896 ch3=84618 mean# 83431.33333 dev# 1878.207893 part# 0.02251202058
160: ch1=87212 ch2=81632 ch3=81450 mean# 83431.33333 dev# 2674.367381 part# 0.03205471223
161: ch1=81156 ch2=84266 ch3=84872 mean# 83431.33333 dev# 1627.81353 part# 0.01951081764
162: ch1=84244 ch2=83354 ch3=82696 mean# 83431.33333 dev# 634.3297425 part# 0.007603015763
163: ch1=81754 ch2=84756 ch3=83784 mean# 83431.33333 dev# 1250.67484 part# 0.01499046929
164: ch1=87058 ch2=81628 ch3=81608 mean# 83431.33333 dev# 2564.453591 part# 0.03073729604
165: ch1=81160 ch2=84262 ch3=84872 mean# 83431.33333 dev# 1625.267431 part# 0.01948030034
166: ch1=84248 ch2=83350 ch3=82696 mean# 83431.33333 dev# 636.2061161 part# 0.007625505799
167: ch1=82190 ch2=84320 ch3=83784 mean# 83431.33333 dev# 904.6197482 part# 0.01084268598
168: ch1=85974 ch2=82712 ch3=81608 mean# 83431.33333 dev# 1853.567611 part# 0.02221668451
169: ch1=81164 ch2=84258 ch3=84872 mean# 83431.33333 dev# 1622.72391 part# 0.01944981394
170: ch1=83960 ch2=83346 ch3=82988 mean# 83431.33333 dev# 401.378735 part# 0.004810887216
171: ch1=84798 ch2=82796 ch3=82700 mean# 83431.33333 dev# 967.1736602 part# 0.0115924512
172: ch1=81160 ch2=84258 ch3=84876 mean# 83431.33333 dev# 1625.771065 part# 0.01948633685
173: ch1=85696 ch2=81898 ch3=82700 mean# 83431.33333 dev# 1634.490203 part# 0.0195908436
174: ch1=80886 ch2=84896 ch3=84512 mean# 83431.33333 dev# 1806.6369 part# 0.02165417749
175: ch1=87504 ch2=81632 ch3=81158 mean# 83431.33333 dev# 2886.304365 part# 0.0345949687
176: ch1=81152 ch2=84266 ch3=84876 mean# 83431.33333 dev# 1630.857716 part# 0.01954730496
177: ch1=84240 ch2=83354 ch3=82700 mean# 83431.33333 dev# 631.0759771 part# 0.007564016442
178: ch1=81610 ch2=83808 ch3=84876 mean# 83431.33333 dev# 1359.680681 part# 0.0162970029
179: ch1=88138 ch2=80544 ch3=81612 mean# 83431.33333 dev# 3356.554715 part# 0.04023134452
180: ch1=82240 ch2=83808 ch3=84246 mean# 83431.33333 dev# 861.1687149 part# 0.01032188604
181: ch1=84194 ch2=85572 ch3=80528 mean# 83431.33333 dev# 2128.650485 part# 0.02551380159
182: ch1=87808 ch2=84134 ch3=78352 mean# 83431.33333 dev# 3892.23923 part# 0.046652008
183: ch1=87050 ch2=88156 ch3=75088 mean# 83431.33333 dev# 5916.880785 part# 0.07091916848
184: ch1=89226 ch2=88156 ch3=72912 mean# 83431.33333 dev# 7451.107539 part# 0.08930826395
185: ch1=90314 ch2=89244 ch3=70736 mean# 83431.33333 dev# 8987.578144 part# 0.107724254
186: ch1=91854 ch2=92056 ch3=66384 mean# 83431.33333 dev# 12054.56708 part# 0.1444848908
187: ch1=94662 ch2=90336 ch3=65296 mean# 83431.33333 dev# 12944.65927 part# 0.1551534508
188: ch1=93574 ch2=94688 ch3=62032 mean# 83431.33333 dev# 15138.44663 part# 0.1814479767
189: ch1=95750 ch2=94688 ch3=59856 mean# 83431.33333 dev# 16675.91511 part# 0.1998759273
190: ch1=98464 ch2=95776 ch3=56054 mean# 83431.33333 dev# 19389.77602 part# 0.2324040052
191: ch1=94528 ch2=100266 ch3=55500 mean# 83431.33333 dev# 19888.86954 part# 0.2383860924
192: ch1=101648 ch2=97498 ch3=51148 mean# 83431.33333 dev# 22890.54875 part# 0.2743639331
193: ch1=100106 ch2=101216 ch3=48972 mean# 83431.33333 dev# 24370.6417 part# 0.2921041859
194: ch1=103370 ch2=101216 ch3=45708 mean# 83431.33333 dev# 26688.91577 part# 0.3198907976
195: ch1=102282 ch2=103392 ch3=44620 mean# 83431.33333 dev# 27447.49802 part# 0.3289830921
196: ch1=104188 ch2=105838 ch3=40268 mean# 83431.33333 dev# 30528.51818 part# 0.3659119058
197: ch1=108810 ch2=103392 ch3=38092 mean# 83431.33333 dev# 32135.96159 part# 0.3851785691
198: ch1=107722 ch2=107744 ch3=34828 mean# 83431.33333 dev# 34367.74776 part# 0.4119285452
199: ch1=106634 ch2=109920 ch3=33740 mean# 83431.33333 dev# 35162.67818 part# 0.4214565053
200: ch1=112532 ch2=108374 ch3=29388 mean# 83431.33333 dev# 38252.09057 part# 0.4584859074
201: ch1=111052 ch2=112322 ch3=26920 mean# 83431.33333 dev# 39962.91048 part# 0.4789916316
202: ch1=114254 ch2=111008 ch3=25032 mean# 83431.33333 dev# 41315.82206 part# 0.4952075008
203: ch1=114712 ch2=114902 ch3=20680 mean# 83431.33333 dev# 44371.96113 part# 0.5318380919
204: ch1=114528 ch2=116174 ch3=19592 mean# 83431.33333 dev# 45146.22678 part# 0.5411183662
205: ch1=119338 ch2=115352 ch3=15604 mean# 83431.33333 dev# 47988.76542 part# 0.575188763
206: ch1=114534 ch2=120524 ch3=15236 mean# 83431.33333 dev# 48283.34869 part# 0.57871961
207: ch1=122336 ch2=117074 ch3=10884 mean# 83431.33333 dev# 51343.67109 part# 0.6154003423
208: ch1=119248 ch2=122338 ch3=8708 mean# 83431.33333 dev# 52852.43251 part# 0.6334842127
209: ch1=124054 ch2=121884 ch3=4356 mean# 83431.33333 dev# 55921.72197 part# 0.6702724232
210: ch1=121878 ch2=124060 ch3=4356 mean# 83431.33333 dev# 55921.79979 part# 0.670273356
211: ch1=123102 ch2=126558 ch3=634 mean# 83431.33333 dev# 58563.55402 part# 0.7019371701
212: ch1=128452 ch2=121842 mean# 125147 dev# 3305 part# 0.02640894308
213: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
214: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
215: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
216: ch1=123146 ch2=127148 mean# 125147 dev# 2001 part# 0.0159891967
217: ch1=127954 ch2=122340 mean# 125147 dev# 2807 part# 0.02242962276
218: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
219: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
220: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
221: ch1=123644 ch2=126650 mean# 125147 dev# 1503 part# 0.01200987639
222: ch1=128544 ch2=121750 mean# 125147 dev# 3397 part# 0.02714407856
223: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
224: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
225: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
226: ch1=123054 ch2=127240 mean# 125147 dev# 2093 part# 0.01672433219
227: ch1=128050 ch2=122244 mean# 125147 dev# 2903 part# 0.02319672066
228: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
229: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
230: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
231: ch1=122464 ch2=127830 mean# 125147 dev# 2683 part# 0.02143878799
232: ch1=128640 ch2=121654 mean# 125147 dev# 3493 part# 0.02791117646
233: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
234: ch1=127326 ch2=122968 mean# 125147 dev# 2179 part# 0.01741152405
235: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
236: ch1=122962 ch2=127332 mean# 125147 dev# 2185 part# 0.01745946767
237: ch1=128142 ch2=122152 mean# 125147 dev# 2995 part# 0.02393185614
238: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
239: ch1=126696 ch2=123598 mean# 125147 dev# 1549 part# 0.01237744413
240: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
241: ch1=122376 ch2=127918 mean# 125147 dev# 2771 part# 0.02214196105
242: ch1=128736 ch2=121558 mean# 125147 dev# 3589 part# 0.02867827435
243: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
244: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
245: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
246: ch1=123962 ch2=126332 mean# 125147 dev# 1185 part# 0.009468864615
247: ch1=128238 ch2=122056 mean# 125147 dev# 3091 part# 0.02469895403
248: ch1=122978 ch2=127316 mean# 125147 dev# 2169 part# 0.01733161802
249: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
250: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
251: ch1=123372 ch2=126922 mean# 125147 dev# 1775 part# 0.01418332042
252: ch1=128828 ch2=121466 mean# 125147 dev# 3681 part# 0.02941340983
253: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
254: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
255: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
256: ch1=123870 ch2=126424 mean# 125147 dev# 1277 part# 0.0102040001
257: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
258: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
259: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
260: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
261: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
262: ch1=127832 ch2=122462 mean# 125147 dev# 2685 part# 0.02145476919
263: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
264: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
265: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
266: ch1=123774 ch2=126520 mean# 125147 dev# 1373 part# 0.01097109799
267: ch1=128422 ch2=121872 mean# 125147 dev# 3275 part# 0.02616922499
268: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
269: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
270: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
271: ch1=123184 ch2=127110 mean# 125147 dev# 1963 part# 0.01568555379
272: ch1=127924 ch2=122370 mean# 125147 dev# 2777 part# 0.02218990467
273: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
274: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
275: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
276: ch1=122594 ch2=127700 mean# 125147 dev# 2553 part# 0.02040000959
277: ch1=121152 ch2=129142 mean# 125147 dev# 3995 part# 0.03192245919
278: ch1=114066 ch2=136228 mean# 125147 dev# 11081 part# 0.08854387241
279: ch1=112520 ch2=137774 mean# 125147 dev# 12627 part# 0.1008973447
280: ch1=107710 ch2=142584 mean# 125147 dev# 17437 part# 0.1393321454
281: ch1=102900 ch2=147394 mean# 125147 dev# 22247 part# 0.1777669461
282: ch1=100960 ch2=149334 mean# 125147 dev# 24187 part# 0.193268716
283: ch1=93284 ch2=157010 mean# 125147 dev# 31863 part# 0.254604585
284: ch1=92826 ch2=157468 mean# 125147 dev# 32321 part# 0.2582642812
285: ch1=88016 ch2=162278 mean# 125147 dev# 37131 part# 0.2966990819
286: ch1=82118 ch2=166520 mean# 124319 dev# 42201 part# 0.3394573637
2025-11-11T17:18:06.471213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518332020376543:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:06.484955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518331951108688:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:06.525879Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518332020376553:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:06.527425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571518332020376543:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:06.539926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518331951108699:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:06.565890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518332020376553:2057] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:06.567094Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518331951108699:2057] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:06.569937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571518332020376543:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:06.570050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:06.570099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518332020376543:2048] [node 2] ICP22 created new session: [1:7571518332020376556:2048]
2025-11-11T17:18:06.570154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518332020376556:2048] [node 2] ICS09 handshake done sender: [1:7571518332020376553:2057] self: [1:7571518332020376554:0] peer: [2:7571518331951108698:0] socket: 17 qp: 18
2025-11-11T17:18:06.570201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518332020376556:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:06.570254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:06.581944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518331951108688:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:18:06.582079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:06.582129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518331951108688:2048] [node 1] ICP22 created new session: [2:7571518331951108702:2048]
2025-11-11T17:18:06.582170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518331951108702:2048] [node 1] ICS09 handshake done sender: [2:7571518331951108699:2057] self: [2:7571518331951108698:0] peer: [1:7571518332020376554:0] socket: 16 qp: 17
2025-11-11T17:18:06.582203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518331951108702:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:06.582234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:06.689323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:06.689545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518331951108702:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-11T17:18:06.689657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518331951108688:2048] [node 1] ICP30 unregister session Session# [2:7571518331951108702:2048] VirtualId# [2:7571518331951108698:0]
2025-11-11T17:18:06.689703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518331951108702:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:06.689764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:06.689824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:06.693169Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:21963) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:18:06.750269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518329346470260:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:06.764755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518333086485226:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:06.779034Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518329346470270:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:06.780620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571518329346470260:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:06.786020Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1588: Handshake [1:7571518329346470270:2057] [node 2] ICH03 handshake failed, explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected
2025-11-11T17:18:06.787802Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1588: Handshake [2:7571518333086485237:2057] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:29719) error from peer: connection checker refused to proceed with incoming connection: rejected
2025-11-11T17:18:06.788799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518333086485226:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:29719) error from peer: connection checker refused to proceed with incoming connection: rejected incoming: [0:0:0] held: no
2025-11-11T17:18:06.788878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518333086485226:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:29719) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:06.788894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:29719) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:06.794097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518333086485226:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:06.844294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518332783604754:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:06.844350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518332783604755:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:06.864422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518330559870865:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:06.864481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518330559870866:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:06.886468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518331812487892:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:06.886517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518331812487893:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:06.935506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518333199587301:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:06.935558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518333199587302:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:06.935575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518333199587303:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:06.947445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518331905717140:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:06.947503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518331905717141:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:06.947537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518331905717142:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:06.969304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518332588115683:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:06.969372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518332588115684:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:06.969400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518332588115685:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:07.063394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518334854838216:2101] [node 5] ICP01 ready to work
2025-11-11T17:18:07.070184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518334854838223:2104] [node 6] ICP01 ready to work
2025-11-11T17:18:07.076043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518336200684454:2062] [node 4] ICP01 ready to work
2025-11-11T17:18:07.098065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518337078572112:2102] [node 5] ICP01 ready to work
2025-11-11T17:18:07.098482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518337078572113:2103] [node 4] ICP01 ready to work
2025-11-11T17:18:07.098767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518337078572114:2104] [node 6] ICP01 ready to work
2025-11-11T17:18:07.097987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518336200684454:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19780051ns) {Configure:104(19775343ns) {ProcessConfigured:135(77396ns) {StartInitialHandshake:157(74062ns) {DropHandshakes:469(3346ns) {DropIncomingHandshake:441(907ns) DropOutgoingHandshake:456(649ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:476(2799ns) {SwitchToState:178(719ns)}}}}}
2025-11-11T17:18:07.099197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518336200684462:2068] [node 6] ICP01 ready to work
2025-11-11T17:18:07.105947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518334854838223:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17642858ns) {Configure:104(17638623ns) {ProcessConfigured:135(11680874ns) {StartInitialHandshake:157(11678831ns) {DropHandshakes:469(3079ns) {DropIncomingHandshake:441(853ns) DropOutgoingHandshake:456(540ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:476(4152ns) {SwitchToState:178(1564ns)}}}}}
2025-11-11T17:18:07.132187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518336883083001:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.144665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518334854838230:2106] [node 4] ICP01 ready to work
2025-11-11T17:18:07.137341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518336883083004:2068] [node 4] ICP01 ready to work
2025-11-11T17:18:07.142702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518331905717142:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(29634318ns) {Configure:104(29632433ns) {ProcessConfigured:135(62359ns) {StartInitialHandshake:157(61093ns) {DropHandshakes:469(3607ns) {DropIncomingHandshake:441(286ns) DropOutgoingHandshake:456(213ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:476(1245ns) {SwitchToState:178(350ns)}}}}}
2025-11-11T17:18:07.150719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083016:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:07.151636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518336107455244:2063] [node 5] ICP01 ready to work
2025-11-11T17:18:07.143167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684463:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
2025-11-11T17:18:07.143239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684464:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.172535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518337078572128:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4431
2025-11-11T17:18:07.166340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518336107455249:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:07.178042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518334854838230:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19852474ns) {Configure:104(19844380ns) {ProcessConfigured:135(72175ns) {StartInitialHandshake:157(67686ns) {DropHandshakes:469(3307ns) {DropIncomingHandshake:441(778ns) DropOutgoingHandshake:456(580ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:476(2948ns) {SwitchToState:178(1003ns)}}}}}
2025-11-11T17:18:07.187356Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518336200684464:2072] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.187545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518331905717140:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.173391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518337078572129:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.176693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518336883083016:2074] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.162269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518333199587301:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(72784568ns) {Configure:104(72779811ns) {ProcessConfigured:135(51989ns) {StartInitialHandshake:157(50332ns) {DropHandshakes:469(2419ns) {DropIncomingHandshake:441(466ns) DropOutgoingHandshake:456(481ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:476(1383ns) {SwitchToState:178(494ns)}}}}}
2025-11-11T17:18:07.189053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518334854838231:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64055
2025-11-11T17:18:07.186894Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572128:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.162787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518337494554637:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:07.191763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518334854838238:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64055
2025-11-11T17:18:07.191223Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572116:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.163088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518337494554638:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.219944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684477:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
2025-11-11T17:18:07.191583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518337078572116:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.230148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518337494554637:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(49684042ns) {Configure:104(49680771ns) {ProcessConfigured:135(13254873ns) {StartInitialHandshake:157(13251062ns) {DropHandshakes:469(2338ns) {DropIncomingHandshake:441(596ns) DropOutgoingHandshake:456(390ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:476(6903ns) {SwitchToState:178(3654ns)}}}}}
2025-11-11T17:18:07.210146Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518337078572116:2109] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.233628Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838231:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.245831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518330559870865:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.237809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518337078572112:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.239933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518337494554639:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
2025-11-11T17:18:07.237936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.250511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518331812487892:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(76930985ns) {Configure:104(76925268ns) {ProcessConfigured:135(47395ns) {StartInitialHandshake:157(46187ns) {DropHandshakes:469(2057ns) {DropIncomingHandshake:441(471ns) DropOutgoingHandshake:456(566ns)} GenerateSessionVirtualId:988(331ns) PrepareNewSessionHandshake:476(3321ns) {SwitchToState:178(252ns)}}}}}
2025-11-11T17:18:07.250878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518336107455255:2068] [node 4] ICP01 ready to work
2025-11-11T17:18:07.238021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518337078572112:2102] [node 5] ICP22 created new session: [1:7571518337078572133:2102]
2025-11-11T17:18:07.253962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518334854838242:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64055
2025-11-11T17:18:07.257851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518336200684463:2071] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.238095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518337078572133:2102] [node 5] ICS09 handshake done sender: [1:7571518337078572116:2109] self: [1:7571518337078572115:0] peer: [5:7571518336200684474:0] socket: 38 qp: -1
2025-11-11T17:18:07.238134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518337078572133:2102] [node 5] ICS10 traffic start
2025-11-11T17:18:07.238176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:07.276870Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518337494554639:2086] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.277097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838242:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.270788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518337078572112:2102] [node 5] ICP35 event processing took too much time PendingConnection:278(27549573ns) {HandleHandshakeStatus:329(27546273ns) {DropIncomingHandshake:441(22041ns) DropOutgoingHandshake:456(370ns) SwitchToState:178(489ns) ProcessPendingSessionEvents:495(27024431ns) {ForwardSessionEventToSession:612(8891462ns) ForwardSessionEventToSession:612(4055247ns) ForwardSessionEventToSession:612(3363ns) ForwardSessionEventToSession:612(1688ns) ForwardSessionEventToSession:612(2505687ns) ForwardSessionEventToSession:612(1909ns) ForwardSessionEventToSession:612(423579ns) ForwardSessionEventToSession:612(18087ns) ForwardSessionEventToSession:612(2818ns) ForwardSessionEventToSession:612(7318363ns)}}}
2025-11-11T17:18:07.319042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518336200684464:2072] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.319684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572118:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.319720Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518337078572118:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.320718Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572129:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.321682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838238:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.319060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518336883083001:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.319455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083032:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:07.328711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518331905717140:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.328800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:07.328858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518331905717140:2048] [node 1] ICP22 created new session: [5:7571518336200684514:2048]
2025-11-11T17:18:07.328896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518336200684514:2048] [node 1] ICS09 handshake done sender: [5:7571518336200684464:2072] self: [5:7571518336200684474:0] peer: [1:7571518337078572115:0] socket: 41 qp: -1
2025-11-11T17:18:07.321096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518337078572135:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.328925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518336200684514:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:07.328973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:07.329572Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:07.334174Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518336200684468:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.334196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518336200684468:2074] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.335156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518331905717140:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:07.335518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684513:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
2025-11-11T17:18:07.335645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518331905717141:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.336286Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518336200684468:2074] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:07.336569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518336200684462:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.336622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:07.336656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518336200684462:2068] [node 6] ICP22 created new session: [5:7571518336200684519:2068]
2025-11-11T17:18:07.336711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518336200684519:2068] [node 6] ICS09 handshake done sender: [5:7571518336200684468:2074] self: [5:7571518336200684467:0] peer: [6:7571518336883083033:0] socket: 45 qp: -1
2025-11-11T17:18:07.336755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518336200684519:2068] [node 6] ICS10 traffic start
2025-11-11T17:18:07.336786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:07.326384Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518336883083032:2090] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.350192Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518336200684513:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.350549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518336200684454:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.390825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
2025-11-11T17:18:07.392410Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518336200684513:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:07.338596Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518337078572118:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:07.414168Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572135:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.414491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518332783604754:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.414556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:07.414599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518332783604754:2048] [node 2] ICP22 created new session: [1:7571518337078572139:2048]
2025-11-11T17:18:07.414643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518337078572139:2048] [node 2] ICS09 handshake done sender: [1:7571518337078572118:2110] self: [1:7571518337078572117:0] peer: [2:7571518334854838241:0] socket: 39 qp: -1
2025-11-11T17:18:07.414675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518337078572139:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:07.414699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:07.349226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518337494554677:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.393526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518336200684454:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.396075Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838218:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.351485Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518337494554659:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.393618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.396124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518334854838218:2102] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.569009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518336107455256:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51326
2025-11-11T17:18:07.569234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518336107455257:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51326
2025-11-11T17:18:07.569422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518336107455258:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51326
2025-11-11T17:18:07.351525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518337494554659:2105] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.485451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518336200684454:2062] [node 4] ICP22 created new session: [5:7571518336200684528:2062]
2025-11-11T17:18:07.490103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518336200684528:2062] [node 4] ICS09 handshake done sender: [5:7571518336200684513:2113] self: [5:7571518336200684524:0] peer: [4:7571518337494554656:0] socket: 66 qp: -1
2025-11-11T17:18:07.497950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518336200684528:2062] [node 4] ICS10 traffic start
2025-11-11T17:18:07.498021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:07.498517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518336200684454:2062] [node 4] ICP35 event processing took too much time PendingConnection:278(104815782ns) {HandleHandshakeStatus:329(104813369ns) {DropIncomingHandshake:441(734ns) DropOutgoingHandshake:456(4580ns) SwitchToState:178(871ns) ProcessPendingSessionEvents:495(127666ns) {ForwardSessionEventToSession:612(85643ns) ForwardSessionEventToSession:612(5952ns) ForwardSessionEventToSession:612(2593ns) ForwardSessionEventToSession:612(1645ns) ForwardSessionEventToSession:612(4019ns) ForwardSessionEventToSession:612(1929ns) ForwardSessionEventToSession:612(2492ns) ForwardSessionEventToSession:612(6549ns) ForwardSessionEventToSession:612(4231ns) ForwardSessionEventToSession:612(4509ns)}}}
2025-11-11T17:18:07.571521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684531:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
2025-11-11T17:18:07.571567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684532:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.585815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518336107455267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51326
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.529948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518332588115684:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.352948Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518337494554659:2105] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:07.530641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083058:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:07.449838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518337494554637:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.449942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:07.449993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518337494554637:2062] [node 5] ICP22 created new session: [4:7571518337494554679:2062]
2025-11-11T17:18:07.450033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518337494554679:2062] [node 5] ICS09 handshake done sender: [4:7571518337494554659:2105] self: [4:7571518337494554656:0] peer: [5:7571518336200684524:0] socket: 60 qp: -1
2025-11-11T17:18:07.450072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518337494554679:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:07.450122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:07.495151Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518337494554677:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.495293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518333199587301:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.439181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518334854838244:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64055
2025-11-11T17:18:07.615473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455267:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.557859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572122:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.562295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083061:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:07.557932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518337078572122:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.615585Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455257:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.562497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083062:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:07.562614Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518336883083016:2074] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:07.562944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518336883083001:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.563040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:07.563080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518336883083001:2063] [node 5] ICP22 created new session: [6:7571518336883083064:2063]
2025-11-11T17:18:07.563127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518336883083064:2063] [node 5] ICS09 handshake done sender: [6:7571518336883083016:2074] self: [6:7571518336883083033:0] peer: [5:7571518336200684467:0] socket: 50 qp: -1
2025-11-11T17:18:07.563158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518336883083064:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:07.563218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:07.594271Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518336883083062:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.594517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083067:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:07.594588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518336883083004:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.661077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838228:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.661150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518334854838228:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.638750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518337494554667:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.638785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518337494554667:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.639721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518337494554683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
2025-11-11T17:18:07.646000Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518337494554667:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.673959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083069:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.694666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518337078572143:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4431
2025-11-11T17:18:07.694892Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518337078572122:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:07.695170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518337078572113:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.695281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:07.695318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518337078572113:2103] [node 4] ICP22 created new session: [1:7571518337078572145:2103]
2025-11-11T17:18:07.695387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518337078572145:2103] [node 4] ICS09 handshake done sender: [1:7571518337078572122:2112] self: [1:7571518337078572121:0] peer: [4:7571518337494554682:0] socket: 68 qp: -1
2025-11-11T17:18:07.695418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518337078572145:2103] [node 4] ICS10 traffic start
2025-11-11T17:18:07.695452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:07.678064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518334854838246:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64055
2025-11-11T17:18:07.718429Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518337494554683:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.718486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518337494554638:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.718570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:07.718643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518337494554638:2063] [node 6] ICP22 created new session: [4:7571518337494554685:2063]
2025-11-11T17:18:07.718713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518337494554685:2063] [node 6] ICS09 handshake done sender: [4:7571518337494554667:2111] self: [4:7571518337494554666:0] peer: [6:7571518336883083068:0] socket: 52 qp: -1
2025-11-11T17:18:07.718750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518337494554685:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:07.718789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:07.719490Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:07.712874Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455258:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.713050Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455256:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.713345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518331812487893:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.713407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518331812487892:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.718042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518336883083061:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.729680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518333199587301:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518337494554677:2116] held: no
2025-11-11T17:18:07.730015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518337494554638:2063] [node 6] ICP18 handshake (actor [4:7571518337494554683:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.730994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518337494554687:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
2025-11-11T17:18:07.731529Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518337494554677:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:07.678248Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518334854838218:2102] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:07.737694Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572143:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.718155Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518336883083067:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.718219Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518336883083062:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:07.741923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518336883083004:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.750464Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518336200684532:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.750579Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518336200684463:2071] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:07.742025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:07.742071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518336883083004:2068] [node 4] ICP22 created new session: [6:7571518336883083070:2068]
2025-11-11T17:18:07.757979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518331905717141:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.758092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:07.758134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518331905717141:2049] [node 2] ICP22 created new session: [5:7571518336200684533:2049]
2025-11-11T17:18:07.758199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518336200684533:2049] [node 2] ICS09 handshake done sender: [5:7571518336200684463:2071] self: [5:7571518336200684517:0] peer: [2:7571518334854838217:0] socket: 37 qp: -1
2025-11-11T17:18:07.758236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518336200684533:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:07.758275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:07.758874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518331905717142:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.766079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518336107455275:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51326
2025-11-11T17:18:07.731804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518333199587301:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.731919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:07.731957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518333199587301:2048] [node 1] ICP22 created new session: [4:7571518337494554689:2048]
2025-11-11T17:18:07.732054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518337494554689:2048] [node 1] ICS09 handshake done sender: [4:7571518337494554677:2116] self: [4:7571518337494554682:0] peer: [1:7571518337078572121:0] socket: 69 qp: -1
2025-11-11T17:18:07.732089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518337494554689:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:07.732141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:07.742147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518336883083070:2068] [node 4] ICS09 handshake done sender: [6:7571518336883083062:2119] self: [6:7571518336883083068:0] peer: [4:7571518337494554666:0] socket: 80 qp: -1
2025-11-11T17:18:07.769880Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518334854838231:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.769990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572120:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.814036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518336200684536:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54515
2025-11-11T17:18:07.822111Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518336200684532:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:07.805777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455260:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.829909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518331905717142:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.830024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:07.830071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518331905717142:2050] [node 3] ICP22 created new session: [5:7571518336200684537:2050]
2025-11-11T17:18:07.830167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518336200684537:2050] [node 3] ICS09 handshake done sender: [5:7571518336200684532:2121] self: [5:7571518336200684535:0] peer: [3:7571518336107455259:0] socket: 86 qp: -1
2025-11-11T17:18:07.830204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518336200684537:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:07.830282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:07.742187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518336883083070:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:07.742258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:07.743111Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:07.807694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518332588115683:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.807786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518332588115685:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:07.807967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518336883083004:2068] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:07.770025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518337078572120:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.805821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518336107455260:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.853679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518334854838216:2101] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.806693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455275:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.807371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518336107455276:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51326
2025-11-11T17:18:07.821683Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518336107455260:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:07.869700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:07.869785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518334854838216:2101] [node 5] ICP22 created new session: [2:7571518334854838250:2101]
2025-11-11T17:18:07.869840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518334854838250:2101] [node 5] ICS09 handshake done sender: [2:7571518334854838218:2102] self: [2:7571518334854838217:0] peer: [5:7571518336200684517:0] socket: 36 qp: -1
2025-11-11T17:18:07.869877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518334854838250:2101] [node 5] ICS10 traffic start
2025-11-11T17:18:07.869916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:07.870561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518334854838216:2101] [node 5] ICP35 event processing took too much time PendingConnection:278(24579105ns) {HandleHandshakeStatus:329(24574461ns) {DropIncomingHandshake:441(1833ns) DropOutgoingHandshake:456(1497ns) SwitchToState:178(731ns) ProcessPendingSessionEvents:495(127894ns) {ForwardSessionEventToSession:612(56087ns) ForwardSessionEventToSession:612(7219ns) ForwardSessionEventToSession:612(2327ns) ForwardSessionEventToSession:612(2068ns) ForwardSessionEventToSession:612(3844ns) ForwardSessionEventToSession:612(1753ns) ForwardSessionEventToSession:612(2012ns) ForwardSessionEventToSession:612(5818ns) ForwardSessionEventToSession:612(27530ns) ForwardSessionEventToSession:612(9025ns)}}}
2025-11-11T17:18:07.870723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838246:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.870823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838233:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.870842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518334854838233:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.871785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518330559870865:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.833808Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518337078572120:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.874938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518337078572124:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.874969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518337078572124:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:07.876974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518332783604755:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.877116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:07.877162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518332783604755:2049] [node 3] ICP22 created new session: [1:7571518337078572149:2049]
2025-11-11T17:18:07.877205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518337078572149:2049] [node 3] ICS09 handshake done sender: [1:7571518337078572120:2111] self: [1:7571518337078572119:0] peer: [3:7571518336107455274:0] socket: 40 qp: -1
2025-11-11T17:18:07.877237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518337078572149:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:07.877264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:07.874555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083074:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:07.871902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:07.871943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518330559870865:2048] [node 1] ICP22 created new session: [2:7571518334854838252:2048]
2025-11-11T17:18:07.871988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518334854838252:2048] [node 1] ICS09 handshake done sender: [2:7571518334854838231:2111] self: [2:7571518334854838241:0] peer: [1:7571518337078572117:0] socket: 46 qp: -1
2025-11-11T17:18:07.872021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518334854838252:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:07.872083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:07.913975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518334854838228:2109] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:07.873916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455262:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:07.873957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518336107455262:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:07.927181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518337494554691:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
2025-11-11T17:18:07.930383Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518337078572124:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:07.930696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518337078572114:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.930815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:07.930869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518337078572114:2104] [node 6] ICP22 created new session: [1:7571518337078572152:2104]
2025-11-11T17:18:07.930944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518337078572152:2104] [node 6] ICS09 handshake done sender: [1:7571518337078572124:2113] self: [1:7571518337078572123:0] peer: [6:7571518336883083072:0] socket: 76 qp: -1
2025-11-11T17:18:07.930979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518337078572152:2104] [node 6] ICS10 traffic start
2025-11-11T17:18:07.931015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:07.926508Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518336883083032:2090] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:07.950250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518334854838216:2101] [node 5] ICP18 handshake (actor [2:7571518334854838246:2119]) is too old
2025-11-11T17:18:07.950402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518334854838223:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.950486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:07.950551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518334854838223:2104] [node 6] ICP22 created new session: [2:7571518334854838256:2104]
2025-11-11T17:18:07.950602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518334854838256:2104] [node 6] ICS09 handshake done sender: [2:7571518334854838228:2109] self: [2:7571518334854838227:0] peer: [6:7571518336883083063:0] socket: 54 qp: -1
2025-11-11T17:18:07.950638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518334854838256:2104] [node 6] ICS10 traffic start
2025-11-11T17:18:07.950666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:07.953780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518332588115684:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:07.953917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:07.953970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518332588115684:2049] [node 2] ICP22 created new session: [6:7571518336883083075:2049]
2025-11-11T17:18:07.954012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518336883083075:2049] [node 2] ICS09 handshake done sender: [6:7571518336883083032:2090] self: [6:7571518336883083063:0] peer: [2:7571518334854838227:0] socket: 65 qp: -1
2025-11-11T17:18:07.954050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518336883083075:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:07.954131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:07.955156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518336107455244:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:07.955298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:07.955341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518336107455244:2063] [node 5] ICP22 created new session: [3:7571518336107455279:2063]
2025-11-11T17:18:07.955383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518336107455279:2063] [node 5] ICS09 handshake done sender: [3:7571518336107455260:2114] self: [3:7571518336107455259:0] peer: [5:7571518336200684535:0] socket: 43 qp: -1
2025-11-11T17:18:07.955417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518336107455279:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:07.955450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:07.956450Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:07.990268Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518334854838233:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:07.993960Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:07.997945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518331812487892:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518336107455256:2111] held: no
2025-11-11T17:18:08.030699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518330559870866:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:08.030803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:08.030850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518330559870866:2049] [node 3] ICP22 created new session: [2:7571518339149805554:2049]
2025-11-11T17:18:08.030922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518339149805554:2049] [node 3] ICS09 handshake done sender: [2:7571518334854838233:2112] self: [2:7571518334854838232:0] peer: [3:7571518336107455273:0] socket: 57 qp: -1
2025-11-11T17:18:08.030953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518339149805554:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:08.030982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:08.038686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518337494554691:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.038813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518333199587302:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:08.039229Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518334854838235:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.039262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518334854838235:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.041053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518341789521989:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
2025-11-11T17:18:08.041305Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518334854838235:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:08.038115Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518336107455256:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:08.052975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518337494554691:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:08.061986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518334854838230:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:08.062111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:08.062160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518334854838230:2106] [node 4] ICP22 created new session: [2:7571518339149805557:2106]
2025-11-11T17:18:08.062211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518339149805557:2106] [node 4] ICS09 handshake done sender: [2:7571518334854838235:2113] self: [2:7571518334854838234:0] peer: [4:7571518341789521988:0] socket: 97 qp: -1
2025-11-11T17:18:08.062282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518339149805557:2106] [node 4] ICS10 traffic start
2025-11-11T17:18:08.062328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.065043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518331812487892:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.065130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:08.065172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518331812487892:2048] [node 1] ICP22 created new session: [3:7571518340402422580:2048]
2025-11-11T17:18:08.065218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518340402422580:2048] [node 1] ICS09 handshake done sender: [3:7571518336107455256:2111] self: [3:7571518336107455274:0] peer: [1:7571518337078572119:0] socket: 62 qp: -1
2025-11-11T17:18:08.065248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518340402422580:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:08.065308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:08.082240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518336107455281:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51326
2025-11-11T17:18:08.102186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518332588115683:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518336883083061:2118] held: no
2025-11-11T17:18:08.102673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518336883083077:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:08.121717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518333199587302:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.121871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:08.121953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518333199587302:2049] [node 2] ICP22 created new session: [4:7571518341789521990:2049]
2025-11-11T17:18:08.122004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518341789521990:2049] [node 2] ICS09 handshake done sender: [4:7571518337494554691:2119] self: [4:7571518341789521988:0] peer: [2:7571518334854838234:0] socket: 98 qp: -1
2025-11-11T17:18:08.122040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518341789521990:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:08.122083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:08.122693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518333199587302:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24720324ns) {HandleHandshakeStatus:329(10634810ns) {DropIncomingHandshake:441(3913ns) DropOutgoingHandshake:456(8757ns) SwitchToState:178(773ns) ProcessPendingSessionEvents:495(109712ns) {ForwardSessionEventToSession:612(67468ns) ForwardSessionEventToSession:612(3500ns) ForwardSessionEventToSession:612(10752ns) ForwardSessionEventToSession:612(2591ns) ForwardSessionEventToSession:612(2543ns) ForwardSessionEventToSession:612(4457ns) ForwardSessionEventToSession:612(1811ns) ForwardSessionEventToSession:612(4719ns) ForwardSessionEventToSession:612(1604ns) ForwardSessionEventToSession:612(1775ns)}}}
2025-11-11T17:18:08.122859Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.126250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518341178050374:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33048
2025-11-11T17:18:08.126318Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518336883083061:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:08.127615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518332588115683:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.127695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:08.127761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518332588115683:2048] [node 1] ICP22 created new session: [6:7571518341178050375:2048]
2025-11-11T17:18:08.127797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518341178050375:2048] [node 1] ICS09 handshake done sender: [6:7571518336883083061:2118] self: [6:7571518336883083072:0] peer: [1:7571518337078572123:0] socket: 79 qp: -1
2025-11-11T17:18:08.127827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518341178050375:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:08.127859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:08.128370Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518336883083067:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:08.128656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518332588115685:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.128743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:08.128808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518332588115685:2050] [node 3] ICP22 created new session: [6:7571518341178050377:2050]
2025-11-11T17:18:08.128850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518341178050377:2050] [node 3] ICS09 handshake done sender: [6:7571518336883083067:2120] self: [6:7571518336883083073:0] peer: [3:7571518336107455261:0] socket: 87 qp: -1
2025-11-11T17:18:08.128903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518341178050377:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:08.128931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:08.126940Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518336107455262:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:08.142409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518333199587302:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:08.154230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518336107455249:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:08.154346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:08.154392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518336107455249:2064] [node 6] ICP22 created new session: [3:7571518340402422583:2064]
2025-11-11T17:18:08.154436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518340402422583:2064] [node 6] ICS09 handshake done sender: [3:7571518336107455262:2115] self: [3:7571518336107455261:0] peer: [6:7571518336883083073:0] socket: 84 qp: -1
2025-11-11T17:18:08.154472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518340402422583:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:08.154538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:08.155639Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518336107455258:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.174449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518339149805559:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64055
2025-11-11T17:18:08.157635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518341789521992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
2025-11-11T17:18:08.182131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518331812487893:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.182236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:08.182305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518331812487893:2049] [node 2] ICP22 created new session: [3:7571518340402422585:2049]
2025-11-11T17:18:08.182361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518340402422585:2049] [node 2] ICS09 handshake done sender: [3:7571518336107455258:2113] self: [3:7571518336107455273:0] peer: [2:7571518334854838232:0] socket: 64 qp: -1
2025-11-11T17:18:08.182394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518340402422585:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:08.182456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:08.196799Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:08.202081Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518341789521992:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.202250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518333199587303:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:08.205768Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518336107455264:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.205810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518336107455264:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.207132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518341789521994:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3186
2025-11-11T17:18:08.213879Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518336107455264:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:08.214550Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518341789521992:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:08.214861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518333199587303:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.214951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:08.214993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518333199587303:2050] [node 3] ICP22 created new session: [4:7571518341789521995:2050]
2025-11-11T17:18:08.215065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518341789521995:2050] [node 3] ICS09 handshake done sender: [4:7571518341789521992:2121] self: [4:7571518341789521993:0] peer: [3:7571518336107455263:0] socket: 59 qp: -1
2025-11-11T17:18:08.215100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518341789521995:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:08.215161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:08.225931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518336107455255:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:08.226042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:08.226093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518336107455255:2068] [node 4] ICP22 created new session: [3:7571518340402422588:2068]
2025-11-11T17:18:08.226133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518340402422588:2068] [node 4] ICS09 handshake done sender: [3:7571518336107455264:2116] self: [3:7571518336107455263:0] peer: [4:7571518341789521993:0] socket: 48 qp: -1
2025-11-11T17:18:08.226164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518340402422588:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:08.226201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:08.269742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:08.270027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518334854838252:2048] [node 1] ICS01 socket: 46 reason# EndOfStream
2025-11-11T17:18:08.270072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518330559870865:2048] [node 1] ICP30 unregister session Session# [2:7571518334854838252:2048] VirtualId# [2:7571518334854838241:0]
2025-11-11T17:18:08.270105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518334854838252:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.270239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:08.270286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:08.281732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:08.281997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518340402422580:2048] [node 1] ICS01 socket: 62 reason# EndOfStream
2025-11-11T17:18:08.282069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518331812487892:2048] [node 1] ICP30 unregister session Session# [3:7571518340402422580:2048] VirtualId# [3:7571518336107455274:0]
2025-11-11T17:18:08.282103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518340402422580:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.282229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:08.282303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:08.291733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:08.292002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518336200684514:2048] [node 1] ICS01 socket: 41 reason# EndOfStream
2025-11-11T17:18:08.292077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518331905717140:2048] [node 1] ICP30 unregister session Session# [5:7571518336200684514:2048] VirtualId# [5:7571518336200684474:0]
2025-11-11T17:18:08.292148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518336200684514:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.292297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:08.292392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:08.293775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:08.294008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518341178050375:2048] [node 1] ICS01 socket: 79 reason# EndOfStream
2025-11-11T17:18:08.294041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518332588115683:2048] [node 1] ICP30 unregister session Session# [6:7571518341178050375:2048] VirtualId# [6:7571518336883083072:0]
2025-11-11T17:18:08.294072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518341178050375:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.294156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:08.294200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:08.298113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:08.298338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518337494554689:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-11-11T17:18:08.298410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518333199587301:2048] [node 1] ICP30 unregister session Session# [4:7571518337494554689:2048] VirtualId# [4:7571518337494554682:0]
2025-11-11T17:18:08.298442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518337494554689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.298541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:08.298591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:08.318401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:08.319143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518340402422585:2049] [node 2] ICS01 socket: 64 reason# EndOfStream
2025-11-11T17:18:08.319188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518331812487893:2049] [node 2] ICP30 unregister session Session# [3:7571518340402422585:2049] VirtualId# [3:7571518336107455273:0]
2025-11-11T17:18:08.319221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518340402422585:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.319350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:08.319412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:08.326497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:08.326795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518336883083075:2049] [node 2] ICS01 socket: 65 reason# EndOfStream
2025-11-11T17:18:08.326839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518332588115684:2049] [node 2] ICP30 unregister session Session# [6:7571518336883083075:2049] VirtualId# [6:7571518336883083063:0]
2025-11-11T17:18:08.326870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518336883083075:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.326985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:08.327076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:08.328935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:08.329153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518336200684533:2049] [node 2] ICS01 socket: 37 reason# EndOfStream
2025-11-11T17:18:08.329200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518331905717141:2049] [node 2] ICP30 unregister session Session# [5:7571518336200684533:2049] VirtualId# [5:7571518336200684517:0]
2025-11-11T17:18:08.329231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518336200684533:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.329348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:08.329393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:08.331223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:08.331421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518341789521990:2049] [node 2] ICS07 socket disconnect 98 reason# EndOfStream
2025-11-11T17:18:08.331466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518341789521990:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.331566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:08.331642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518341789521990:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:08.332729Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14330) Couldn't connect to any resolved address
2025-11-11T17:18:08.333036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518333199587302:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14330) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:08.333083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518341789521990:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:08.333135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518333199587302:2049] [node 2] ICP30 unregister session Session# [4:7571518341789521990:2049] VirtualId# [4:7571518341789521988:0]
2025-11-11T17:18:08.333184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:08.333209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518333199587302:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:08.333225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:08.344311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518333199587302:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:08.370093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:08.370338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518336200684537:2050] [node 3] ICS01 socket: 86 reason# EndOfStream
2025-11-11T17:18:08.370376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518331905717142:2050] [node 3] ICP30 unregister session Session# [5:7571518336200684537:2050] VirtualId# [5:7571518336200684535:0]
2025-11-11T17:18:08.370439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518336200684537:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.370558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:08.370605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:08.369812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:08.370067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518341178050377:2050] [node 3] ICS01 socket: 87 reason# EndOfStream
2025-11-11T17:18:08.370104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518332588115685:2050] [node 3] ICP30 unregister session Session# [6:7571518341178050377:2050] VirtualId# [6:7571518336883083073:0]
2025-11-11T17:18:08.370147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518341178050377:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.370245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:08.370291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:08.370420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:08.370621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518341789521995:2050] [node 3] ICS01 socket: 59 reason# EndOfStream
2025-11-11T17:18:08.370660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518333199587303:2050] [node 3] ICP30 unregister session Session# [4:7571518341789521995:2050] VirtualId# [4:7571518341789521993:0]
2025-11-11T17:18:08.370681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518341789521995:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.370750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:08.370788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:08.401832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:08.402055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518336883083070:2068] [node 4] ICS01 socket: 80 reason# EndOfStream
2025-11-11T17:18:08.402099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518336883083004:2068] [node 4] ICP30 unregister session Session# [6:7571518336883083070:2068] VirtualId# [6:7571518336883083068:0]
2025-11-11T17:18:08.402131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518336883083070:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.402533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:08.402615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:08.403100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:08.408420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518336200684528:2062] [node 4] ICS07 socket disconnect 66 reason# EndOfStream
2025-11-11T17:18:08.408463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518336200684528:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.408595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:08.408664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518336200684528:2062] [node 4] ICS15 start handshake
2025-11-11T17:18:08.409879Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29196) Couldn't connect to any resolved address
2025-11-11T17:18:08.410193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518336200684454:2062] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:08.410241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518336200684528:2062] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:08.410306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518336200684454:2062] [node 4] ICP30 unregister session Session# [5:7571518336200684528:2062] VirtualId# [5:7571518336200684524:0]
2025-11-11T17:18:08.410355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:08.410382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518336200684454:2062] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:08.410403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:08.437783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:08.438025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518336883083064:2063] [node 5] ICS01 socket: 50 reason# EndOfStream
2025-11-11T17:18:08.438067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518336883083001:2063] [node 5] ICP30 unregister session Session# [6:7571518336883083064:2063] VirtualId# [6:7571518336883083033:0]
2025-11-11T17:18:08.438096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518336883083064:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:08.438225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:08.438276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:08.491528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518339350516728:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:08.491570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518339350516729:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:08.514612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518340267437010:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:08.521189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518340267437011:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:08.534667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518338581906600:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:08.535019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518338581906599:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:08.552308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518338503331702:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:08.552347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518338503331703:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:08.552361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518338503331704:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:08.571099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518339581059293:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:08.571135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518339581059294:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:08.571156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518339581059295:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:08.589505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518339654911385:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:08.589531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518339654911386:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:08.589548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518339654911387:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:08.721172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518339350516790:2108] [node 5] ICP01 ready to work
2025-11-11T17:18:08.741573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518339654911443:2090] [node 4] ICP01 ready to work
2025-11-11T17:18:08.746755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518339581059355:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:08.746873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518339350516728:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24913787ns) {Configure:104(24909112ns) {ProcessConfigured:135(1535840ns) {StartInitialHandshake:157(1534050ns) {DropHandshakes:469(1649ns) {DropIncomingHandshake:441(389ns) DropOutgoingHandshake:456(241ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:476(5159ns) {SwitchToState:178(1268ns)}}}}}
2025-11-11T17:18:08.747074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518339350516794:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:08.752880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518338581906662:2107] [node 5] ICP01 ready to work
2025-11-11T17:18:08.754356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518339654911449:2094] [node 5] ICP01 ready to work
2025-11-11T17:18:08.756733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518338581906663:2108] [node 4] ICP01 ready to work
2025-11-11T17:18:08.757281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518339350516797:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:08.767314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518340267437081:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:08.770121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518338581906666:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:08.761150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518339654911443:2090] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17024740ns) {Configure:104(17020511ns) {ProcessConfigured:135(48155ns) {StartInitialHandshake:157(43809ns) {DropHandshakes:469(1114ns) {DropIncomingHandshake:441(344ns) DropOutgoingHandshake:456(252ns)} GenerateSessionVirtualId:988(500ns) PrepareNewSessionHandshake:476(1222ns) {SwitchToState:178(461ns)}}}}}
2025-11-11T17:18:08.774420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518340267437082:2114] [node 5] ICP01 ready to work
2025-11-11T17:18:08.790996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518339581059361:2110] [node 6] ICP01 ready to work
2025-11-11T17:18:08.817663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518339581059295:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(44400406ns) {Configure:104(44391921ns) {ProcessConfigured:135(8195880ns) {StartInitialHandshake:157(8193073ns) {DropHandshakes:469(2921ns) {DropIncomingHandshake:441(899ns) DropOutgoingHandshake:456(418ns)} GenerateSessionVirtualId:988(866ns) PrepareNewSessionHandshake:476(5204ns) {SwitchToState:178(2901ns)}}}}}
2025-11-11T17:18:08.825600Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518338581906600:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(57682571ns) {Configure:104(57679373ns) {ProcessConfigured:135(8616933ns) {StartInitialHandshake:157(4130433ns) {DropHandshakes:469(4616ns) {DropIncomingHandshake:441(563ns) DropOutgoingHandshake:456(743ns)} GenerateSessionVirtualId:988(2598ns) PrepareNewSessionHandshake:476(3410ns) {SwitchToState:178(1418ns)}}}}}
2025-11-11T17:18:08.830605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518340267437083:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.833414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518339581059293:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39777328ns) {Configure:104(39773774ns) {ProcessConfigured:135(2582048ns) {StartInitialHandshake:157(2579728ns) {DropHandshakes:469(1382ns) {DropIncomingHandshake:441(472ns) DropOutgoingHandshake:456(268ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:476(2547ns) {SwitchToState:178(1238ns)}}}}}
2025-11-11T17:18:08.835728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518339350516807:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12107
2025-11-11T17:18:08.831265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518340267437010:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(43438296ns) {Configure:104(43430507ns) {ProcessConfigured:135(1177881ns) {StartInitialHandshake:157(1176033ns) {DropHandshakes:469(1618ns) {DropIncomingHandshake:441(613ns) DropOutgoingHandshake:456(284ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:476(2011ns) {SwitchToState:178(895ns)}}}}}
2025-11-11T17:18:08.853158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518339581059294:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25211059ns) {Configure:104(25203084ns) {ProcessConfigured:135(2607768ns) {StartInitialHandshake:157(2603699ns) {DropHandshakes:469(2885ns) {DropIncomingHandshake:441(612ns) DropOutgoingHandshake:456(549ns)} GenerateSessionVirtualId:988(799ns) PrepareNewSessionHandshake:476(4170ns) {SwitchToState:178(1981ns)}}}}}
2025-11-11T17:18:08.834904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518338503331704:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(78222258ns) {Configure:104(78220311ns) {ProcessConfigured:135(235911ns) {StartInitialHandshake:157(231200ns) {DropHandshakes:469(546ns) {DropIncomingHandshake:441(143ns) DropOutgoingHandshake:456(32ns)} GenerateSessionVirtualId:988(38ns) PrepareNewSessionHandshake:476(1547ns) {SwitchToState:178(588ns)}}}}}
2025-11-11T17:18:08.843375Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518338503331702:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(82444298ns) {Configure:104(80164060ns) {ProcessConfigured:135(3906428ns) {StartInitialHandshake:157(3904781ns) {DropHandshakes:469(1728ns) {DropIncomingHandshake:441(435ns) DropOutgoingHandshake:456(349ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:476(2545ns) {SwitchToState:178(710ns)}}}}}
2025-11-11T17:18:08.855631Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516807:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.857919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518339581059367:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:08.858030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518339581059368:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.858105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518339581059369:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:08.855102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518338503331703:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18557745ns) {Configure:104(18322259ns) {ProcessConfigured:135(909957ns) {StartInitialHandshake:157(907977ns) {DropHandshakes:469(3346ns) {DropIncomingHandshake:441(838ns) DropOutgoingHandshake:456(666ns)} GenerateSessionVirtualId:988(918ns) PrepareNewSessionHandshake:476(2407ns) {SwitchToState:178(781ns)}}}}}
2025-11-11T17:18:08.856999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518340267437081:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23040503ns) {Configure:104(23037217ns) {ProcessConfigured:135(3656799ns) {StartInitialHandshake:157(3654594ns) {DropHandshakes:469(2482ns) {DropIncomingHandshake:441(495ns) DropOutgoingHandshake:456(562ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:476(2049ns) {SwitchToState:178(1891ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.863574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518340267437086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54636
2025-11-11T17:18:08.863724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518340267437087:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54636
2025-11-11T17:18:08.863819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518340267437088:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54636
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.864938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518338503331771:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:08.866917Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437087:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.867012Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437088:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.867062Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437086:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.865667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518338581906678:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5153
2025-11-11T17:18:08.866141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518338581906679:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5153
2025-11-11T17:18:08.868967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518338503331772:2108] [node 5] ICP01 ready to work
2025-11-11T17:18:08.865235Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518339581059368:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.875412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518340267437082:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18182089ns) {Configure:104(18177064ns) {ProcessConfigured:135(58073ns) {StartInitialHandshake:157(55735ns) {DropHandshakes:469(4631ns) {DropIncomingHandshake:441(537ns) DropOutgoingHandshake:456(289ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:476(1947ns) {SwitchToState:178(538ns)}}}}}
2025-11-11T17:18:08.875590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518340267437010:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:08.877234Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516793:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.865538Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518339581059369:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.866590Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518339581059367:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.876459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518339581059295:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:08.876884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518339581059293:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:08.877266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518339350516793:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:08.881787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518338581906668:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.887758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518338503331771:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16967983ns) {Configure:104(16963846ns) {ProcessConfigured:135(4816045ns) {StartInitialHandshake:157(4813390ns) {DropHandshakes:469(2330ns) {DropIncomingHandshake:441(429ns) DropOutgoingHandshake:456(452ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:476(808805ns) {SwitchToState:178(1981ns)}}}}}
2025-11-11T17:18:08.881830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518338581906668:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:08.898365Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516796:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.887425Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518338581906678:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.902636Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518338581906668:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:08.898408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518339350516796:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.921639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518339350516809:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12107
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.922022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518339581059377:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:08.923537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518339350516810:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12107
2025-11-11T17:18:08.922650Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518339581059369:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:08.922793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518339581059295:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.928483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:08.924482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518338503331772:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40271366ns) {Configure:104(40263463ns) {ProcessConfigured:135(584393ns) {StartInitialHandshake:157(580049ns) {DropHandshakes:469(3375ns) {DropIncomingHandshake:441(769ns) DropOutgoingHandshake:456(705ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:476(2494ns) {SwitchToState:178(906ns)}}}}}
2025-11-11T17:18:08.941782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518339581059295:2050] [node 3] ICP22 created new session: [5:7571518339581059378:2050]
2025-11-11T17:18:08.941848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518339581059378:2050] [node 3] ICS09 handshake done sender: [5:7571518339581059369:2116] self: [5:7571518339581059375:0] peer: [3:7571518338581906667:0] socket: 58 qp: -1
2025-11-11T17:18:08.941889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518339581059378:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:08.941935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.960623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518340267437103:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54636
2025-11-11T17:18:08.923735Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518339350516793:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:08.928562Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518339350516796:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:08.964937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518340267437086:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:08.965297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518340267437010:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:08.965387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:08.965440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518340267437010:2048] [node 1] ICP22 created new session: [2:7571518340267437104:2048]
2025-11-11T17:18:08.965480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518340267437104:2048] [node 1] ICS09 handshake done sender: [2:7571518340267437086:2119] self: [2:7571518340267437102:0] peer: [1:7571518339350516792:0] socket: 53 qp: -1
2025-11-11T17:18:08.965512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518340267437104:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:08.965543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.969767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518340267437106:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54636
2025-11-11T17:18:08.956808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518338581906662:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:08.973713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:08.980119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518339350516728:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:08.968425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518338503331773:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.981921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518338581906662:2107] [node 5] ICP22 created new session: [3:7571518338581906683:2107]
2025-11-11T17:18:08.980248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:08.968581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518338503331774:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
2025-11-11T17:18:08.996555Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437092:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.980316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518339350516728:2048] [node 2] ICP22 created new session: [1:7571518339350516813:2048]
2025-11-11T17:18:08.989342Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518338581906679:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.968680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518338503331776:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
2025-11-11T17:18:08.996594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518340267437092:2123] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:08.989946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518338581906600:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:08.969208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518338503331781:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
2025-11-11T17:18:08.980368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518339350516813:2048] [node 2] ICS09 handshake done sender: [1:7571518339350516793:2110] self: [1:7571518339350516792:0] peer: [2:7571518340267437102:0] socket: 43 qp: -1
2025-11-11T17:18:08.999402Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518340267437092:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:08.990038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518338581906684:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5153
2025-11-11T17:18:09.004734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518340267437011:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:08.980399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518339350516813:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.978546Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518338503331776:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.008030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518339581059295:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(55021830ns) {HandleHandshakeStatus:329(55018777ns) {DropIncomingHandshake:441(1076ns) DropOutgoingHandshake:456(6687ns) SwitchToState:178(7231ns) ProcessPendingSessionEvents:495(20826251ns) {ForwardSessionEventToSession:612(7476034ns) ForwardSessionEventToSession:612(9515ns) ForwardSessionEventToSession:612(2343ns) ForwardSessionEventToSession:612(2114ns) ForwardSessionEventToSession:612(3013830ns) ForwardSessionEventToSession:612(2490806ns) ForwardSessionEventToSession:612(3916099ns) ForwardSessionEventToSession:612(3086ns) ForwardSessionEventToSession:612(2778ns) ForwardSessionEventToSession:612(3889371ns)}}}
2025-11-11T17:18:08.998223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518338581906683:2107] [node 5] ICS09 handshake done sender: [3:7571518338581906668:2111] self: [3:7571518338581906667:0] peer: [5:7571518339581059375:0] socket: 50 qp: -1
2025-11-11T17:18:09.004836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:08.980450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.039562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343876026676:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:08.984397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518338503331703:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:08.999153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518338581906683:2107] [node 5] ICS10 traffic start
2025-11-11T17:18:09.004876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518340267437011:2049] [node 3] ICP22 created new session: [2:7571518344562404404:2049]
2025-11-11T17:18:09.003427Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518338503331781:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.981070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518339350516790:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.001765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:09.007917Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518338581906674:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:08.981126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:09.004925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518344562404404:2049] [node 3] ICS09 handshake done sender: [2:7571518340267437092:2123] self: [2:7571518340267437091:0] peer: [3:7571518338581906685:0] socket: 65 qp: -1
2025-11-11T17:18:09.003540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518338503331775:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.053069Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518339581059367:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:08.981179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518339350516790:2108] [node 5] ICP22 created new session: [1:7571518339350516815:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.003731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518338503331704:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.007939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518338581906674:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.004954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518344562404404:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.981221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518339350516815:2108] [node 5] ICS09 handshake done sender: [1:7571518339350516796:2111] self: [1:7571518339350516795:0] peer: [5:7571518339581059376:0] socket: 44 qp: -1
2025-11-11T17:18:09.061964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518339581059293:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.008053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518338503331773:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.005011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:09.034048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518338581906662:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(92091945ns) {HandleHandshakeStatus:329(92088971ns) {DropIncomingHandshake:441(28484ns) DropOutgoingHandshake:456(420ns) SwitchToState:178(435ns) ProcessPendingSessionEvents:495(11692346ns) {ForwardSessionEventToSession:612(11645899ns) ForwardSessionEventToSession:612(11060ns) ForwardSessionEventToSession:612(2401ns) ForwardSessionEventToSession:612(6130ns) ForwardSessionEventToSession:612(1898ns) ForwardSessionEventToSession:612(1714ns) ForwardSessionEventToSession:612(4366ns) ForwardSessionEventToSession:612(2236ns) ForwardSessionEventToSession:612(1780ns) ForwardSessionEventToSession:612(3884ns)}}}
2025-11-11T17:18:08.981247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518339350516815:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:09.075485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878760:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.022415Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518338503331775:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.034463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518338581906687:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5153
2025-11-11T17:18:09.005476Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437090:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.080569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:08.981276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.062478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518342798299085:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
2025-11-11T17:18:09.086921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518339581059293:2048] [node 1] ICP22 created new session: [5:7571518343876026678:2048]
2025-11-11T17:18:09.086258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518343949878760:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.052672Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.005507Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518340267437090:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.075804Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518338503331776:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:09.094425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518339654911386:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.096189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878759:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.038948Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516809:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.062581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518339350516817:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12107
2025-11-11T17:18:09.079848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518339350516790:2108] [node 5] ICP18 handshake (actor [1:7571518339350516809:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.089875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518343876026678:2048] [node 1] ICS09 handshake done sender: [5:7571518339581059367:2114] self: [5:7571518339581059376:0] peer: [1:7571518339350516795:0] socket: 56 qp: -1
2025-11-11T17:18:09.064858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518338581906600:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518338581906679:2116] held: no
2025-11-11T17:18:09.080338Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516810:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.057565Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437106:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.096782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518338503331703:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.063637Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518340267437090:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:09.071935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518344562404408:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54636
2025-11-11T17:18:09.091142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518340267437081:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.091230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:09.091303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518340267437081:2107] [node 4] ICP22 created new session: [2:7571518344562404411:2107]
2025-11-11T17:18:09.091340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518344562404411:2107] [node 4] ICS09 handshake done sender: [2:7571518340267437090:2122] self: [2:7571518340267437089:0] peer: [4:7571518338503331785:0] socket: 64 qp: -1
2025-11-11T17:18:09.091371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518344562404411:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:09.091446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:09.096281Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344562404408:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.104241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437099:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.104284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518340267437099:2125] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.089904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518343876026678:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:09.105236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878762:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.106172Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518340267437099:2125] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.065036Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518338581906684:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.107744Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518343949878759:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.102119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518339350516728:2048] [node 2] ICP18 handshake (actor [1:7571518339350516810:2117]) is too old
2025-11-11T17:18:09.105161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:09.093253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:09.108813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516801:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.108835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518339350516801:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.107916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518339654911385:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.106347Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518340267437094:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.080156Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518338581906679:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:09.096396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518339581059293:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(31549759ns) {HandleHandshakeStatus:329(31546453ns) {DropIncomingHandshake:441(2153ns) DropOutgoingHandshake:456(11216ns) SwitchToState:178(641ns) ProcessPendingSessionEvents:495(75819ns) {ForwardSessionEventToSession:612(40732ns) ForwardSessionEventToSession:612(6870ns) ForwardSessionEventToSession:612(4062ns) ForwardSessionEventToSession:612(1725ns) ForwardSessionEventToSession:612(1509ns) ForwardSessionEventToSession:612(3268ns) ForwardSessionEventToSession:612(1485ns) ForwardSessionEventToSession:612(3815ns) ForwardSessionEventToSession:612(1939ns) ForwardSessionEventToSession:612(4095ns)}}}
2025-11-11T17:18:09.088970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518338581906600:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.106496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518338503331703:2049] [node 2] ICP22 created new session: [4:7571518342798299086:2049]
2025-11-11T17:18:09.121219Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518339350516801:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.098564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343876026679:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:09.106383Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518340267437094:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.149693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878764:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.108712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518342798299086:2049] [node 2] ICS09 handshake done sender: [4:7571518338503331776:2116] self: [4:7571518338503331785:0] peer: [2:7571518340267437089:0] socket: 71 qp: -1
2025-11-11T17:18:09.120762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518342798299086:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:09.121155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:09.132973Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.134840Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8468) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.119281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518340267437082:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.089044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:09.121529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.102731Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518343876026679:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.089081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518338581906600:2049] [node 2] ICP22 created new session: [3:7571518342876873986:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.155839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518343949878762:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.162160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518339654911387:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.162291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.154206Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516817:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.154423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518339350516797:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.154493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:09.154569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518339350516797:2109] [node 6] ICP22 created new session: [1:7571518343645484117:2109]
2025-11-11T17:18:09.154614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518343645484117:2109] [node 6] ICS09 handshake done sender: [1:7571518339350516801:2113] self: [1:7571518339350516800:0] peer: [6:7571518343949878763:0] socket: 86 qp: -1
2025-11-11T17:18:09.154642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518343645484117:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:09.154690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:09.121582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518340267437082:2114] [node 5] ICP22 created new session: [2:7571518344562404415:2114]
2025-11-11T17:18:09.089117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518342876873986:2049] [node 2] ICS09 handshake done sender: [3:7571518338581906679:2116] self: [3:7571518338581906685:0] peer: [2:7571518340267437091:0] socket: 66 qp: -1
2025-11-11T17:18:09.103952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518339581059294:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.121648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518344562404415:2114] [node 5] ICS09 handshake done sender: [2:7571518340267437099:2125] self: [2:7571518340267437098:0] peer: [5:7571518343876026682:0] socket: 94 qp: -1
2025-11-11T17:18:09.121679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518344562404415:2114] [node 5] ICS10 traffic start
2025-11-11T17:18:09.121710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:09.161848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518340267437094:2124] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:09.186184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518340267437083:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.186285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:09.186327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518340267437083:2109] [node 6] ICP22 created new session: [2:7571518344562404418:2109]
2025-11-11T17:18:09.186413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518344562404418:2109] [node 6] ICS09 handshake done sender: [2:7571518340267437094:2124] self: [2:7571518340267437093:0] peer: [6:7571518343949878761:0] socket: 92 qp: -1
2025-11-11T17:18:09.186451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518344562404418:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:09.186494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:09.166209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518338503331704:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8468) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518338503331781:2119] held: no
2025-11-11T17:18:09.119211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343876026683:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:09.155627Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.166745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518343876026679:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:09.190602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518339581059294:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518343876026679:2120] held: no
2025-11-11T17:18:09.190721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518339581059294:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.190800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:09.190848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518339581059294:2049] [node 2] ICP22 created new session: [5:7571518343876026684:2049]
2025-11-11T17:18:09.190893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518343876026684:2049] [node 2] ICS09 handshake done sender: [5:7571518343876026679:2120] self: [5:7571518343876026682:0] peer: [2:7571518340267437098:0] socket: 60 qp: -1
2025-11-11T17:18:09.190924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518343876026684:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:09.190954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:09.193837Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.196156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518343949878764:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.196304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518339654911386:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518343949878760:2116] held: no
2025-11-11T17:18:09.196542Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518343949878760:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:09.192048Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516806:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.089147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518342876873986:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:09.166529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518342798299088:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
2025-11-11T17:18:09.170107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518338503331703:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(73077486ns) {HandleHandshakeStatus:329(73074989ns) {DropIncomingHandshake:441(456ns) DropOutgoingHandshake:456(3416ns) SwitchToState:178(1311ns) ProcessPendingSessionEvents:495(44864620ns) {ForwardSessionEventToSession:612(30597237ns) ForwardSessionEventToSession:612(5159ns) ForwardSessionEventToSession:612(6158507ns) ForwardSessionEventToSession:612(3846ns) ForwardSessionEventToSession:612(870454ns) ForwardSessionEventToSession:612(7172766ns) ForwardSessionEventToSession:612(3589ns) ForwardSessionEventToSession:612(2265ns) ForwardSessionEventToSession:612(5105ns) ForwardSessionEventToSession:612(25821ns)}}}
2025-11-11T17:18:09.170323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518338503331703:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:09.193292Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518338503331781:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.203780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518340267437082:2114] [node 5] ICP35 event processing took too much time PendingConnection:278(79274223ns) {HandleHandshakeStatus:329(79271085ns) {DropIncomingHandshake:441(15766ns) DropOutgoingHandshake:456(2955ns) SwitchToState:178(546ns) ProcessPendingSessionEvents:495(32544460ns) {ForwardSessionEventToSession:612(27286218ns) ForwardSessionEventToSession:612(6822ns) ForwardSessionEventToSession:612(1083813ns) ForwardSessionEventToSession:612(2837ns) ForwardSessionEventToSession:612(2504ns) ForwardSessionEventToSession:612(2868343ns) ForwardSessionEventToSession:612(3317ns) ForwardSessionEventToSession:612(1270080ns) ForwardSessionEventToSession:612(3880ns) ForwardSessionEventToSession:612(2880ns)}}}
2025-11-11T17:18:09.089182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:09.192092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518339350516806:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.198113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518338503331704:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.204493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.198245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:09.202146Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518339350516806:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:09.209879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518339654911386:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.213973Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518339581059373:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.092740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518342876873985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5153
2025-11-11T17:18:09.214005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518339581059373:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.225858Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518339581059373:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:09.226259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518339581059361:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.226371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:09.226425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518339581059361:2110] [node 6] ICP22 created new session: [5:7571518343876026688:2110]
2025-11-11T17:18:09.226468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518343876026688:2110] [node 6] ICS09 handshake done sender: [5:7571518339581059373:2117] self: [5:7571518339581059372:0] peer: [6:7571518343949878771:0] socket: 82 qp: -1
2025-11-11T17:18:09.226499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518343876026688:2110] [node 6] ICS10 traffic start
2025-11-11T17:18:09.226539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:09.198286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518338503331704:2050] [node 3] ICP22 created new session: [4:7571518342798299091:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.209966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:09.096230Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518338581906674:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.238031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518339350516729:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.210005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518339654911386:2049] [node 2] ICP22 created new session: [6:7571518343949878769:2049]
2025-11-11T17:18:09.210082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518343949878769:2049] [node 2] ICS09 handshake done sender: [6:7571518343949878760:2116] self: [6:7571518343949878761:0] peer: [2:7571518340267437093:0] socket: 63 qp: -1
2025-11-11T17:18:09.210114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518343949878769:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:09.210177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:09.210668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518339654911449:2094] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.210762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878768:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.215527Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518343949878768:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.215688Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518343949878759:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:09.242053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518339654911385:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.242134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:09.242197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518339654911385:2048] [node 1] ICP22 created new session: [6:7571518343949878774:2048]
2025-11-11T17:18:09.242238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518343949878774:2048] [node 1] ICS09 handshake done sender: [6:7571518343949878759:2115] self: [6:7571518343949878763:0] peer: [1:7571518339350516800:0] socket: 87 qp: -1
2025-11-11T17:18:09.242270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518343949878774:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:09.242310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:09.243239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878773:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.244379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518339654911443:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.244660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878776:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.099483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518338581906663:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.198321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518342798299091:2050] [node 3] ICS09 handshake done sender: [4:7571518338503331781:2119] self: [4:7571518342798299082:0] peer: [3:7571518338581906673:0] socket: 78 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.105193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:09.272869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:09.198346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518342798299091:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:09.282339Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.283121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343876026686:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:09.106712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518338581906663:2108] [node 4] ICP22 created new session: [3:7571518342876873990:2108]
2025-11-11T17:18:09.198378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:09.283226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518339350516729:2049] [node 3] ICP22 created new session: [1:7571518343645484121:2049]
2025-11-11T17:18:09.284472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518339654911449:2094] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518343949878764:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.286280Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518339350516799:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.248878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518338503331778:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.119384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518342876873990:2108] [node 4] ICS09 handshake done sender: [3:7571518338581906674:2114] self: [3:7571518338581906673:0] peer: [4:7571518342798299082:0] socket: 77 qp: -1
2025-11-11T17:18:09.287749Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518343876026686:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.286301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518339350516799:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.287451Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518339350516799:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:09.287666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518343645484121:2049] [node 3] ICS09 handshake done sender: [1:7571518339350516806:2114] self: [1:7571518339350516805:0] peer: [3:7571518342876873992:0] socket: 88 qp: -1
2025-11-11T17:18:09.288694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518343645484121:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:09.298451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:09.298879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518339350516794:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.298958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:09.298996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518339350516794:2107] [node 4] ICP22 created new session: [1:7571518343645484124:2107]
2025-11-11T17:18:09.299031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518343645484124:2107] [node 4] ICS09 handshake done sender: [1:7571518339350516799:2112] self: [1:7571518339350516798:0] peer: [4:7571518342798299094:0] socket: 45 qp: -1
2025-11-11T17:18:09.299089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518343645484124:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:09.299121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.284704Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518343949878762:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:09.248908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518338503331778:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.285404Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518343949878764:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:09.285636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518339654911387:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.288642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518339654911449:2094] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.289521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:09.289566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518339654911387:2050] [node 3] ICP22 created new session: [6:7571518343949878779:2050]
2025-11-11T17:18:09.289660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518343949878779:2050] [node 3] ICS09 handshake done sender: [6:7571518343949878762:2117] self: [6:7571518343949878767:0] peer: [3:7571518338581906671:0] socket: 61 qp: -1
2025-11-11T17:18:09.289738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518343949878779:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:09.289790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:09.291545Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2367) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.292709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518339654911443:2090] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2367) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518343949878768:2121] held: no
2025-11-11T17:18:09.292861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518343949878778:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41766
2025-11-11T17:18:09.293277Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518343949878768:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:09.293492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518339654911443:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.293554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:09.294123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518339654911443:2090] [node 4] ICP22 created new session: [6:7571518343949878781:2090]
2025-11-11T17:18:09.299863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518343949878781:2090] [node 4] ICS09 handshake done sender: [6:7571518343949878768:2121] self: [6:7571518343949878777:0] peer: [4:7571518338503331777:0] socket: 100 qp: -1
2025-11-11T17:18:09.314010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:09.314031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518343949878781:2090] [node 4] ICS10 traffic start
2025-11-11T17:18:09.314556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:09.119446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518342876873990:2108] [node 4] ICS10 traffic start
2025-11-11T17:18:09.298439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518339581059355:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.306697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343876026691:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14894
2025-11-11T17:18:09.119504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:09.253862Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518338503331778:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:09.281702Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518338503331774:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.281834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518338503331702:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.287586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518338503331771:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.288779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:09.288841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518338503331771:2109] [node 6] ICP22 created new session: [4:7571518342798299096:2109]
2025-11-11T17:18:09.288878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518342798299096:2109] [node 6] ICS09 handshake done sender: [4:7571518338503331778:2117] self: [4:7571518338503331777:0] peer: [6:7571518343949878777:0] socket: 54 qp: -1
2025-11-11T17:18:09.288904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518342798299096:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:09.288934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:09.302393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518338503331780:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.302414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518338503331780:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.303031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518342798299095:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16137
2025-11-11T17:18:09.314188Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518338503331774:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:09.314450Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.318848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518338503331780:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:09.347471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518338503331702:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.347597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:09.347638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518338503331702:2048] [node 1] ICP22 created new session: [4:7571518342798299099:2048]
2025-11-11T17:18:09.347680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518342798299099:2048] [node 1] ICS09 handshake done sender: [4:7571518338503331774:2114] self: [4:7571518342798299094:0] peer: [1:7571518339350516798:0] socket: 69 qp: -1
2025-11-11T17:18:09.347942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518342798299099:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:09.347986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:09.341708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518339654911449:2094] [node 5] ICP22 created new session: [6:7571518343949878782:2094]
2025-11-11T17:18:09.322265Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518343876026686:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:09.322490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518339581059355:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.322586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:09.322627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518339581059355:2107] [node 4] ICP22 created new session: [5:7571518343876026692:2107]
2025-11-11T17:18:09.322668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518343876026692:2107] [node 4] ICS09 handshake done sender: [5:7571518343876026686:2122] self: [5:7571518343876026690:0] peer: [4:7571518338503331779:0] socket: 101 qp: -1
2025-11-11T17:18:09.322702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518343876026692:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:09.322735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:09.120018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518338581906663:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(20837238ns) {HandleHandshakeStatus:329(20835538ns) {DropIncomingHandshake:441(16645ns) DropOutgoingHandshake:456(580ns) SwitchToState:178(933ns) ProcessPendingSessionEvents:495(85445ns) {ForwardSessionEventToSession:612(47765ns) ForwardSessionEventToSession:612(5317ns) ForwardSessionEventToSession:612(1615ns) ForwardSessionEventToSession:612(6788ns) ForwardSessionEventToSession:612(3918ns) ForwardSessionEventToSession:612(1640ns) ForwardSessionEventToSession:612(3903ns) ForwardSessionEventToSession:612(1458ns) ForwardSessionEventToSession:612(1850ns) ForwardSessionEventToSession:612(4470ns)}}}
2025-11-11T17:18:09.356784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518338503331772:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.356904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:09.341785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518343949878782:2094] [node 5] ICS09 handshake done sender: [6:7571518343949878764:2118] self: [6:7571518343949878771:0] peer: [5:7571518339581059372:0] socket: 96 qp: -1
2025-11-11T17:18:09.356952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518338503331772:2108] [node 5] ICP22 created new session: [4:7571518342798299101:2108]
2025-11-11T17:18:09.357026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518342798299101:2108] [node 5] ICS09 handshake done sender: [4:7571518338503331780:2118] self: [4:7571518338503331779:0] peer: [5:7571518343876026690:0] socket: 55 qp: -1
2025-11-11T17:18:09.341829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518343949878782:2094] [node 5] ICS10 traffic start
2025-11-11T17:18:09.357058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518342798299101:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:09.341874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:09.357091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:09.357021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518339654911443:2090] [node 4] ICP35 event processing took too much time PendingConnection:278(63406202ns) {HandleHandshakeStatus:329(63405090ns) {DropIncomingHandshake:441(612ns) DropOutgoingHandshake:456(234ns) SwitchToState:178(336ns) ProcessPendingSessionEvents:495(994ns)}}
2025-11-11T17:18:09.357925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518339654911449:2094] [node 5] ICP35 event processing took too much time PendingConnection:278(69180055ns) {HandleHandshakeStatus:329(69177521ns) {DropIncomingHandshake:441(1310ns) DropOutgoingHandshake:456(321ns) SwitchToState:178(285ns) ProcessPendingSessionEvents:495(873ns)}}
2025-11-11T17:18:09.359437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518339581059355:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(36688491ns) {HandleHandshakeStatus:329(36686544ns) {DropIncomingHandshake:441(964ns) DropOutgoingHandshake:456(4100ns) SwitchToState:178(890ns) ProcessPendingSessionEvents:495(427209ns) {ForwardSessionEventToSession:612(389347ns) ForwardSessionEventToSession:612(2603ns) ForwardSessionEventToSession:612(8580ns) ForwardSessionEventToSession:612(2044ns) ForwardSessionEventToSession:612(4030ns) ForwardSessionEventToSession:612(1722ns) ForwardSessionEventToSession:612(1641ns) ForwardSessionEventToSession:612(3824ns) ForwardSessionEventToSession:612(1869ns) ForwardSessionEventToSession:612(2008ns)}}}
2025-11-11T17:18:09.156790Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518342876873985:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.184796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518338581906599:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.196964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518338581906599:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19517738ns) {IncomingHandshake:261(19515867ns) {DropIncomingHandshake:441(465ns) PrepareNewSessionHandshake:476(1469ns) {SwitchToState:178(521ns)} GenerateSessionVirtualId:988(529ns) IssueIncomingHandshakeReply:193(19494800ns) {SwitchToState:178(2482ns)}}}
2025-11-11T17:18:09.201772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518342876873993:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5153
2025-11-11T17:18:09.202334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518338581906672:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.202362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518338581906672:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.227476Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518338581906672:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:09.245895Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518342876873985:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:09.252964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518338581906666:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.253041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:09.253079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518338581906666:2109] [node 6] ICP22 created new session: [3:7571518342876873996:2109]
2025-11-11T17:18:09.253183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518342876873996:2109] [node 6] ICS09 handshake done sender: [3:7571518338581906672:2113] self: [3:7571518338581906671:0] peer: [6:7571518343949878767:0] socket: 51 qp: -1
2025-11-11T17:18:09.253222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518342876873996:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:09.253257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:09.258136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518342876873995:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5153
2025-11-11T17:18:09.258332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518338581906599:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.258413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:09.258457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518338581906599:2048] [node 1] ICP22 created new session: [3:7571518342876873998:2048]
2025-11-11T17:18:09.272925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518342876873998:2048] [node 1] ICS09 handshake done sender: [3:7571518342876873985:2119] self: [3:7571518342876873992:0] peer: [1:7571518339350516805:0] socket: 93 qp: -1
2025-11-11T17:18:09.279208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518342876873998:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:09.284887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:09.329969Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518342876873995:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.330277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518338581906666:2109] [node 6] ICP18 handshake (actor [3:7571518342876873995:2121]) is too old
2025-11-11T17:18:09.359978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518338581906599:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(97923601ns) {HandleHandshakeStatus:329(97919069ns) {DropIncomingHandshake:441(683ns) DropOutgoingHandshake:456(4589ns) SwitchToState:178(519ns) ProcessPendingSessionEvents:495(52581795ns) {ForwardSessionEventToSession:612(10465703ns) ForwardSessionEventToSession:612(463129ns) ForwardSessionEventToSession:612(5479969ns) ForwardSessionEventToSession:612(21061792ns) ForwardSessionEventToSession:612(6284ns) ForwardSessionEventToSession:612(1991ns) ForwardSessionEventToSession:612(938092ns) ForwardSessionEventToSession:612(7735629ns) ForwardSessionEventToSession:612(5789ns) ForwardSessionEventToSession:612(1933753ns)}}}
2025-11-11T17:18:09.363396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518339350516729:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(123694972ns) {HandleHandshakeStatus:329(123691168ns) {DropIncomingHandshake:441(17439ns) DropOutgoingHandshake:456(639ns) SwitchToState:178(686ns) ProcessPendingSessionEvents:495(30933411ns) {ForwardSessionEventToSession:612(18975933ns) ForwardSessionEventToSession:612(5120ns) ForwardSessionEventToSession:612(4645762ns) ForwardSessionEventToSession:612(1904595ns) ForwardSessionEventToSession:612(2128ns) ForwardSessionEventToSession:612(1877ns) ForwardSessionEventToSession:612(2498009ns) ForwardSessionEventToSession:612(2074778ns) ForwardSessionEventToSession:612(806729ns) ForwardSessionEventToSession:612(3514ns)}}}
2025-11-11T17:18:09.363779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518338503331702:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:09.385567Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:09.385791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518340267437104:2048] [node 1] ICS01 socket: 53 reason# EndOfStream
2025-11-11T17:18:09.385826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518340267437010:2048] [node 1] ICP30 unregister session Session# [2:7571518340267437104:2048] VirtualId# [2:7571518340267437102:0]
2025-11-11T17:18:09.385855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518340267437104:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.385932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:09.385979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:09.386316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:09.386526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518342876873998:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-11T17:18:09.386547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518338581906599:2048] [node 1] ICP30 unregister session Session# [3:7571518342876873998:2048] VirtualId# [3:7571518342876873992:0]
2025-11-11T17:18:09.386566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518342876873998:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.386625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:09.386658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:09.389719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:09.389923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518343876026678:2048] [node 1] ICS07 socket disconnect 56 reason# EndOfStream
2025-11-11T17:18:09.389967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518343876026678:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.390041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:09.390078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518343876026678:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:09.391130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:09.391295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518343949878774:2048] [node 1] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-11T17:18:09.391314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518343949878774:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.391367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:09.391407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518343949878774:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:09.391895Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address
2025-11-11T17:18:09.390542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:09.397800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518339581059293:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:09.390746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518342798299099:2048] [node 1] ICS07 socket disconnect 69 reason# EndOfStream
2025-11-11T17:18:09.390777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518342798299099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.390831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:09.390870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518342798299099:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:09.391940Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address
2025-11-11T17:18:09.392281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518338503331702:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:09.392324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518342798299099:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:09.392355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518338503331702:2048] [node 1] ICP30 unregister session Session# [4:7571518342798299099:2048] VirtualId# [4:7571518342798299094:0]
2025-11-11T17:18:09.392390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:09.392415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518338503331702:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.392425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.402879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518338503331702:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:09.406302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518343876026678:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:09.406351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518339581059293:2048] [node 1] ICP30 unregister session Session# [5:7571518343876026678:2048] VirtualId# [5:7571518339581059376:0]
2025-11-11T17:18:09.406404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:09.406437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518339581059293:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.406449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.406555Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address
2025-11-11T17:18:09.406706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518339654911385:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:09.406763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518343949878774:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:09.406853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518339654911385:2048] [node 1] ICP30 unregister session Session# [6:7571518343949878774:2048] VirtualId# [6:7571518343949878763:0]
2025-11-11T17:18:09.407316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:09.407359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518339654911385:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.407371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19247) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.413743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518339581059293:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:09.417739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518339654911385:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:09.432542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:09.432727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518342876873986:2049] [node 2] ICS01 socket: 66 reason# EndOfStream
2025-11-11T17:18:09.432793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518338581906600:2049] [node 2] ICP30 unregister session Session# [3:7571518342876873986:2049] VirtualId# [3:7571518338581906685:0]
2025-11-11T17:18:09.432835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518342876873986:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.432914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:09.432961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:09.436755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:09.436951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518343949878769:2049] [node 2] ICS01 socket: 63 reason# EndOfStream
2025-11-11T17:18:09.436981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518339654911386:2049] [node 2] ICP30 unregister session Session# [6:7571518343949878769:2049] VirtualId# [6:7571518343949878761:0]
2025-11-11T17:18:09.437002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518343949878769:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.437053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:09.437085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:09.435125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:09.435363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518342798299086:2049] [node 2] ICS07 socket disconnect 71 reason# EndOfStream
2025-11-11T17:18:09.437532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:09.435386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518342798299086:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.435441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:09.435475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518342798299086:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:09.436186Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Couldn't connect to any resolved address
2025-11-11T17:18:09.436439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518338503331703:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:09.436486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518342798299086:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:09.437826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518343876026684:2049] [node 2] ICS01 socket: 60 reason# EndOfStream
2025-11-11T17:18:09.436515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518338503331703:2049] [node 2] ICP30 unregister session Session# [4:7571518342798299086:2049] VirtualId# [4:7571518338503331785:0]
2025-11-11T17:18:09.436568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:09.437845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518339581059294:2049] [node 2] ICP30 unregister session Session# [5:7571518343876026684:2049] VirtualId# [5:7571518343876026682:0]
2025-11-11T17:18:09.436613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518338503331703:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.437859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518343876026684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.436626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:09.437902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:09.437931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:09.445740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518338503331703:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:09.467546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:09.467780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518343949878779:2050] [node 3] ICS01 socket: 61 reason# EndOfStream
2025-11-11T17:18:09.467822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518339654911387:2050] [node 3] ICP30 unregister session Session# [6:7571518343949878779:2050] VirtualId# [6:7571518343949878767:0]
2025-11-11T17:18:09.468052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518343949878779:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.468144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:09.468219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:09.468446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:09.468679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518342798299091:2050] [node 3] ICS01 socket: 78 reason# EndOfStream
2025-11-11T17:18:09.468715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518338503331704:2050] [node 3] ICP30 unregister session Session# [4:7571518342798299091:2050] VirtualId# [4:7571518342798299082:0]
2025-11-11T17:18:09.468792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518342798299091:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.468900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:09.468938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:09.469615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:09.469794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518339581059378:2050] [node 3] ICS01 socket: 58 reason# EndOfStream
2025-11-11T17:18:09.469834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518339581059295:2050] [node 3] ICP30 unregister session Session# [5:7571518339581059378:2050] VirtualId# [5:7571518339581059375:0]
2025-11-11T17:18:09.469883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518339581059378:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.469984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:09.470089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:09.499564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:09.499791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518343876026692:2107] [node 4] ICS01 socket: 101 reason# EndOfStream
2025-11-11T17:18:09.499835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518339581059355:2107] [node 4] ICP30 unregister session Session# [5:7571518343876026692:2107] VirtualId# [5:7571518343876026690:0]
2025-11-11T17:18:09.499896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518343876026692:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.499992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:09.500032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:09.500300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:09.500648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518343949878781:2090] [node 4] ICS01 socket: 100 reason# EndOfStream
2025-11-11T17:18:09.500680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518339654911443:2090] [node 4] ICP30 unregister session Session# [6:7571518343949878781:2090] VirtualId# [6:7571518343949878777:0]
2025-11-11T17:18:09.500700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518343949878781:2090] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.500766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:09.500802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:09.521390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:09.521661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518343949878782:2094] [node 5] ICS01 socket: 96 reason# EndOfStream
2025-11-11T17:18:09.521690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518339654911449:2094] [node 5] ICP30 unregister session Session# [6:7571518343949878782:2094] VirtualId# [6:7571518343949878771:0]
2025-11-11T17:18:09.521713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518343949878782:2094] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:09.521758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:09.521787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:09.574029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518343432963976:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:09.574560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518343432963977:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:09.594198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518344545316278:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:09.594224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518344545316279:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:09.616701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518343939704259:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:09.616755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518343939704260:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:09.638914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518343391530783:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:09.638964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518343391530785:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:09.639703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518343391530784:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:09.659239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518343471714791:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:09.659295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518343471714792:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:09.659970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518343471714793:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:09.680156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518345064280518:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:09.680216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518345064280520:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:09.680604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518345064280519:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:09.800175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518343432964039:2108] [node 5] ICP01 ready to work
2025-11-11T17:18:09.810296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518345064280579:2104] [node 5] ICP01 ready to work
2025-11-11T17:18:09.823342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518345064280582:2108] [node 4] ICP01 ready to work
2025-11-11T17:18:09.829188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518343432964040:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:09.837453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518343939704324:2108] [node 5] ICP01 ready to work
2025-11-11T17:18:09.836828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518344545316342:2108] [node 5] ICP01 ready to work
2025-11-11T17:18:09.841536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518343471714859:2108] [node 4] ICP01 ready to work
2025-11-11T17:18:09.837453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518343432964041:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:09.837121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518344545316341:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:09.841635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518343939704323:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:09.842843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518343391530846:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:09.843082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518343391530847:2108] [node 5] ICP01 ready to work
2025-11-11T17:18:09.844322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518344545316343:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:09.839286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518343432963976:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(78852720ns) {Configure:104(78844863ns) {ProcessConfigured:135(36991ns) {StartInitialHandshake:157(35065ns) {DropHandshakes:469(1558ns) {DropIncomingHandshake:441(369ns) DropOutgoingHandshake:456(285ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:476(861ns) {SwitchToState:178(95ns)}}}}}
2025-11-11T17:18:09.849993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518343471714860:2110] [node 6] ICP01 ready to work
2025-11-11T17:18:09.838213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518345064280519:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21899083ns) {Configure:104(21895754ns) {ProcessConfigured:135(1266265ns) {StartInitialHandshake:157(1264227ns) {DropHandshakes:469(879975ns) {DropIncomingHandshake:441(828ns) DropOutgoingHandshake:456(1093ns)} GenerateSessionVirtualId:988(1067ns) PrepareNewSessionHandshake:476(1605ns) {SwitchToState:178(544ns)}}}}}
2025-11-11T17:18:09.850457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518343939704329:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:09.851012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518345064280579:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16781528ns) {Configure:104(16772366ns) {ProcessConfigured:135(43561ns) {StartInitialHandshake:157(41122ns) {DropHandshakes:469(1386ns) {DropIncomingHandshake:441(284ns) DropOutgoingHandshake:456(240ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:476(1014ns) {SwitchToState:178(340ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.865743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518343432963977:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16128774ns) {Configure:104(16126203ns) {ProcessConfigured:135(2739351ns) {StartInitialHandshake:157(2737525ns) {DropHandshakes:469(2551ns) {DropIncomingHandshake:441(540ns) DropOutgoingHandshake:456(734ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:476(3481ns) {SwitchToState:178(1781ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.879233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518344545316279:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(41858152ns) {Configure:104(41849597ns) {ProcessConfigured:135(11458801ns) {StartInitialHandshake:157(11454657ns) {DropHandshakes:469(3418ns) {DropIncomingHandshake:441(894ns) DropOutgoingHandshake:456(515ns)} GenerateSessionVirtualId:988(1251ns) PrepareNewSessionHandshake:476(5146ns) {SwitchToState:178(1677ns)}}}}}
2025-11-11T17:18:09.882180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518344545316348:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63012
2025-11-11T17:18:09.901519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518343471714791:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(57995501ns) {Configure:104(57990238ns) {ProcessConfigured:135(55715ns) {StartInitialHandshake:157(53508ns) {DropHandshakes:469(2913ns) {DropIncomingHandshake:441(806ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(639ns) PrepareNewSessionHandshake:476(2347ns) {SwitchToState:178(609ns)}}}}}
2025-11-11T17:18:09.900133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518343432964055:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-11T17:18:09.882346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518344545316349:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63012
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.905962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964055:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.902115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518343939704329:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(48704626ns) {Configure:104(48701120ns) {ProcessConfigured:135(9595883ns) {StartInitialHandshake:157(8162058ns) {DropHandshakes:469(4726ns) {DropIncomingHandshake:441(1236ns) DropOutgoingHandshake:456(461ns)} GenerateSessionVirtualId:988(2296ns) PrepareNewSessionHandshake:476(3012ns) {SwitchToState:178(1116ns)}}}}}
2025-11-11T17:18:09.906524Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964043:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.897078Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316348:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.905034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518344545316278:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.906404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518345064280520:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(58988440ns) {Configure:104(58320760ns) {ProcessConfigured:135(31582768ns) {StartInitialHandshake:157(31579943ns) {DropHandshakes:469(2502ns) {DropIncomingHandshake:441(406ns) DropOutgoingHandshake:456(444ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:476(3379ns) {SwitchToState:178(973ns)}}}}}
2025-11-11T17:18:09.906553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518343432964043:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.907171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518343939704339:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16745
2025-11-11T17:18:09.929918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316349:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.929951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518344545316341:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(62471678ns) {Configure:104(62467243ns) {ProcessConfigured:135(2811683ns) {StartInitialHandshake:157(2809627ns) {DropHandshakes:469(781ns) {DropIncomingHandshake:441(276ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(332ns) PrepareNewSessionHandshake:476(1684ns) {SwitchToState:178(484ns)}}}}}
2025-11-11T17:18:09.933662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518344545316342:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(70989202ns) {Configure:104(70984178ns) {ProcessConfigured:135(5035601ns) {StartInitialHandshake:157(4753646ns) {DropHandshakes:469(1907ns) {DropIncomingHandshake:441(378ns) DropOutgoingHandshake:456(211ns)} GenerateSessionVirtualId:988(1024ns) PrepareNewSessionHandshake:476(3441ns) {SwitchToState:178(912ns)}}}}}
2025-11-11T17:18:09.933795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518343471714792:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(66434166ns) {Configure:104(66201246ns) {ProcessConfigured:135(2339920ns) {StartInitialHandshake:157(2338860ns) {DropHandshakes:469(1315ns) {DropIncomingHandshake:441(320ns) DropOutgoingHandshake:456(213ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:476(1909ns) {SwitchToState:178(905ns)}}}}}
2025-11-11T17:18:09.909774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518345064280591:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:09.907927Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518343432964043:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:09.908666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518343391530785:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(57392834ns) {Configure:104(57388972ns) {ProcessConfigured:135(2328710ns) {StartInitialHandshake:157(2327143ns) {DropHandshakes:469(1602ns) {DropIncomingHandshake:441(359ns) DropOutgoingHandshake:456(223ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:476(2258ns) {SwitchToState:178(477ns)}}}}}
2025-11-11T17:18:09.929935Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518345064280591:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.909521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518343432963976:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.933659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518345064280518:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.937771Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518343939704339:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.950555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518343939704260:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.914138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518343391530783:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(63628664ns) {Configure:104(63619171ns) {ProcessConfigured:135(4958968ns) {StartInitialHandshake:157(4956642ns) {DropHandshakes:469(1658ns) {DropIncomingHandshake:441(398ns) DropOutgoingHandshake:456(306ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:476(2192ns) {SwitchToState:178(475ns)}}}}}
2025-11-11T17:18:09.913689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518343432964056:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-11T17:18:09.951967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343471714868:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
2025-11-11T17:18:09.937843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518343391530784:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(72197149ns) {Configure:104(71820115ns) {ProcessConfigured:135(2813878ns) {StartInitialHandshake:157(2811812ns) {DropHandshakes:469(2664ns) {DropIncomingHandshake:441(481ns) DropOutgoingHandshake:456(586ns)} GenerateSessionVirtualId:988(883ns) PrepareNewSessionHandshake:476(1904ns) {SwitchToState:178(444ns)}}}}}
2025-11-11T17:18:09.921846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:09.978439Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316345:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.954276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518343391530846:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20367554ns) {Configure:104(18695154ns) {ProcessConfigured:135(63135ns) {StartInitialHandshake:157(60770ns) {DropHandshakes:469(2263ns) {DropIncomingHandshake:441(685ns) DropOutgoingHandshake:456(372ns)} GenerateSessionVirtualId:988(633ns) PrepareNewSessionHandshake:476(1697ns) {SwitchToState:178(763ns)}}}}}
2025-11-11T17:18:09.952155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343471714869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
2025-11-11T17:18:09.978476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518344545316345:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-11T17:18:09.925874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518343432963976:2048] [node 2] ICP22 created new session: [1:7571518343432964060:2048]
2025-11-11T17:18:09.952272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343471714870:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.955972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518343391530854:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.983257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518345064280598:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:09.933658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518343432964060:2048] [node 2] ICS09 handshake done sender: [1:7571518343432964043:2110] self: [1:7571518343432964042:0] peer: [2:7571518344545316355:0] socket: 48 qp: -1
2025-11-11T17:18:09.985818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518343939704342:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16745
2025-11-11T17:18:09.956138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518343391530855:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
2025-11-11T17:18:09.969271Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518343471714868:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.984419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518344545316357:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63012
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.934293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518343432964060:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:09.986109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518345064280599:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:09.971713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518343471714869:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.956222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518343391530856:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
2025-11-11T17:18:09.981370Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518343391530854:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.987334Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518344545316345:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:09.987482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518343939704341:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16745
2025-11-11T17:18:09.989764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518345064280600:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:09.983088Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518343391530855:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.937968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:09.988690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518344545316361:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63012
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.981147Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518343471714870:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.949949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964045:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.982047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518343471714793:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(112497902ns) {Configure:104(112492900ns) {ProcessConfigured:135(15644260ns) {StartInitialHandshake:157(15641432ns) {DropHandshakes:469(3691ns) {DropIncomingHandshake:441(811ns) DropOutgoingHandshake:456(636ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:476(3927ns) {SwitchToState:178(1972ns)}}}}}
2025-11-11T17:18:09.990064Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518345064280591:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:09.995691Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316357:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.995814Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316351:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.995832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518344545316351:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.996310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518344545316279:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.996365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:09.996407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518344545316279:2049] [node 3] ICP22 created new session: [2:7571518344545316365:2049]
2025-11-11T17:18:09.996473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518344545316365:2049] [node 3] ICS09 handshake done sender: [2:7571518344545316345:2110] self: [2:7571518344545316344:0] peer: [3:7571518343939704340:0] socket: 58 qp: -1
2025-11-11T17:18:09.996502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518344545316365:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:09.996559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.986565Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518343391530856:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.953263Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518343432964045:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.983210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518343471714859:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(78486746ns) {Configure:104(78482419ns) {ProcessConfigured:135(17658533ns) {StartInitialHandshake:157(17656245ns) {DropHandshakes:469(3268ns) {DropIncomingHandshake:441(680ns) DropOutgoingHandshake:456(661ns)} GenerateSessionVirtualId:988(1199ns) PrepareNewSessionHandshake:476(732097ns) {SwitchToState:178(2075ns)}}}}}
2025-11-11T17:18:09.997238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518344545316362:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63012
2025-11-11T17:18:09.990244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518345064280518:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.997367Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518343939704342:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.990864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518343471714791:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.958894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518343432964062:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-11T17:18:09.997327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518343471714793:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.000847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518343471714877:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
2025-11-11T17:18:09.958995Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518343432964045:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:09.986685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518343391530783:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.990294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:09.998632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518343939704259:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:09.997784Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518344545316348:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:09.973669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518343432964040:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:09.988323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518343391530784:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.003646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518343939704331:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.002636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518344545316278:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:09.991057Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518345064280599:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.003674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518343939704331:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.010191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518343939704343:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16745
2025-11-11T17:18:09.979332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:09.998786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518343391530867:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
2025-11-11T17:18:09.991152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518345064280518:2048] [node 1] ICP22 created new session: [6:7571518345064280601:2048]
2025-11-11T17:18:10.005250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:09.999280Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518343391530854:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:10.018526Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:09.981363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518343432964040:2109] [node 6] ICP22 created new session: [1:7571518343432964064:2109]
2025-11-11T17:18:09.999465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518343391530783:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.032136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518348234671641:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16745
2025-11-11T17:18:10.032354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518348234671643:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16745
2025-11-11T17:18:10.013903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518344545316278:2048] [node 1] ICP22 created new session: [2:7571518348840283664:2048]
2025-11-11T17:18:09.983163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518343432964064:2109] [node 6] ICS09 handshake done sender: [1:7571518343432964045:2111] self: [1:7571518343432964044:0] peer: [6:7571518345064280597:0] socket: 49 qp: -1
2025-11-11T17:18:09.991197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518345064280601:2048] [node 1] ICS09 handshake done sender: [6:7571518345064280591:2114] self: [6:7571518345064280597:0] peer: [1:7571518343432964044:0] socket: 55 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.999551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:10.032798Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518343939704331:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:09.983979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518343432964064:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:10.025860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518348840283664:2048] [node 1] ICS09 handshake done sender: [2:7571518344545316348:2112] self: [2:7571518344545316355:0] peer: [1:7571518343432964042:0] socket: 56 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:09.999587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518343391530783:2048] [node 1] ICP22 created new session: [4:7571518343391530868:2048]
2025-11-11T17:18:09.991229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518345064280601:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:10.027171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518348840283664:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:10.033310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518343939704260:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518343939704339:2116] held: no
2025-11-11T17:18:10.034378Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518348234671641:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.991301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:10.028072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:09.984019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:09.999641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518343391530868:2048] [node 1] ICS09 handshake done sender: [4:7571518343391530854:2113] self: [4:7571518343391530864:0] peer: [1:7571518343432964046:0] socket: 69 qp: -1
2025-11-11T17:18:10.029199Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518344545316351:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:09.985258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964056:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.999666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518343391530868:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:09.997309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518345064280582:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.045570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518343939704324:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.045337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518349359247902:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:09.999694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:09.988111Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964047:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.045657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:10.033128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518344545316343:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.052314Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518349359247902:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:09.988130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518343432964047:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.033201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:10.001732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518343391530858:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.045717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518343939704324:2108] [node 5] ICP22 created new session: [3:7571518348234671645:2108]
2025-11-11T17:18:10.052518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518345064280519:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.045755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518348234671645:2108] [node 5] ICS09 handshake done sender: [3:7571518343939704331:2113] self: [3:7571518343939704330:0] peer: [5:7571518343471714880:0] socket: 53 qp: -1
2025-11-11T17:18:10.001764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518343391530858:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:09.992061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518343432963976:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(73700177ns) {HandleHandshakeStatus:329(73697350ns) {DropIncomingHandshake:441(412ns) DropOutgoingHandshake:456(491ns) SwitchToState:178(507ns) ProcessPendingSessionEvents:495(26306445ns) {ForwardSessionEventToSession:612(22410829ns) ForwardSessionEventToSession:612(812866ns) ForwardSessionEventToSession:612(471158ns) ForwardSessionEventToSession:612(2967ns) ForwardSessionEventToSession:612(822421ns) ForwardSessionEventToSession:612(2300ns) ForwardSessionEventToSession:612(590493ns) ForwardSessionEventToSession:612(3006ns) ForwardSessionEventToSession:612(1906ns) ForwardSessionEventToSession:612(1169158ns)}}}
2025-11-11T17:18:10.034343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518344545316343:2107] [node 4] ICP22 created new session: [2:7571518348840283669:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.068362Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:10.034396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518348840283669:2107] [node 4] ICS09 handshake done sender: [2:7571518344545316351:2114] self: [2:7571518344545316350:0] peer: [4:7571518343391530865:0] socket: 60 qp: -1
2025-11-11T17:18:10.045788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518348234671645:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:10.063953Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518343391530858:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:09.997367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518343432964047:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:10.034433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518348840283669:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:10.069623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518349359247901:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.045825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:10.001607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518343432963976:2048] [node 2] ICP18 handshake (actor [1:7571518343432964056:2116]) is too old
2025-11-11T17:18:10.078427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518347686498168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
2025-11-11T17:18:10.034472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:10.042772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518348840283665:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63012
2025-11-11T17:18:10.005127Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964053:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.079831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518345064280519:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518349359247902:2120] held: no
2025-11-11T17:18:10.108526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518349359247904:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:10.129146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518345064280519:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(32341386ns) {HandleHandshakeStatus:395(26233954ns) {LogHandshakeFail:472(21459ns) {UpdateErrorStateLog:544(5085ns)} DropOutgoingHandshake:456(3309ns) ProcessPendingSessionEvents:495(22054565ns) {DropSessionEvent:512(13934ns) DropSessionEvent:512(3869ns) DropSessionEvent:512(8480359ns) DropSessionEvent:512(1959991ns) DropSessionEvent:512(12766ns) DropSessionEvent:512(3212ns) DropSessionEvent:512(3864933ns) DropSessionEvent:512(12247ns) DropSessionEvent:512(7679060ns) DropSessionEvent:512(5774ns)}}}
2025-11-11T17:18:10.129694Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518345064280599:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:10.129998Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518349359247902:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:10.138259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518349359247905:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:10.138427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518345064280519:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.138504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:10.138555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518345064280519:2049] [node 2] ICP22 created new session: [6:7571518349359247906:2049]
2025-11-11T17:18:10.138600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518349359247906:2049] [node 2] ICS09 handshake done sender: [6:7571518349359247902:2120] self: [6:7571518349359247903:0] peer: [2:7571518344545316356:0] socket: 105 qp: -1
2025-11-11T17:18:10.046384Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518343939704339:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:10.054731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518343939704260:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.054800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:10.054837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518343939704260:2049] [node 2] ICP22 created new session: [3:7571518348234671648:2049]
2025-11-11T17:18:10.054878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518348234671648:2049] [node 2] ICS09 handshake done sender: [3:7571518343939704339:2116] self: [3:7571518343939704340:0] peer: [2:7571518344545316344:0] socket: 62 qp: -1
2025-11-11T17:18:10.054920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518348234671648:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:10.054974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:10.068097Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518343939704342:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:10.079681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518343939704259:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.090298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:10.098545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518343939704259:2048] [node 1] ICP22 created new session: [3:7571518348234671651:2048]
2025-11-11T17:18:10.102049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518348234671651:2048] [node 1] ICS09 handshake done sender: [3:7571518343939704342:2118] self: [3:7571518343939704344:0] peer: [1:7571518343432964051:0] socket: 91 qp: -1
2025-11-11T17:18:10.105898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518348234671651:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:10.110966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:10.149708Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518343939704338:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.149736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518343939704338:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.005165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518343432964053:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.151725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518343391530846:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.053441Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316359:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.053463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518344545316359:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.054266Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518344545316359:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:10.066646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518344545316341:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.066713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:10.066755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518344545316341:2109] [node 6] ICP22 created new session: [2:7571518348840283673:2109]
2025-11-11T17:18:10.066797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518348840283673:2109] [node 6] ICS09 handshake done sender: [2:7571518344545316359:2116] self: [2:7571518344545316356:0] peer: [6:7571518349359247903:0] socket: 52 qp: -1
2025-11-11T17:18:10.066836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518348840283673:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:10.066898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:10.098611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518344545316278:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(63534647ns) {HandleHandshakeStatus:329(63532296ns) {DropIncomingHandshake:441(896ns) DropOutgoingHandshake:456(4075ns) SwitchToState:178(143ns) ProcessPendingSessionEvents:495(17768845ns) {ForwardSessionEventToSession:612(14947548ns) ForwardSessionEventToSession:612(737830ns) ForwardSessionEventToSession:612(382546ns) ForwardSessionEventToSession:612(2141ns) ForwardSessionEventToSession:612(1130097ns) ForwardSessionEventToSession:612(3773ns) ForwardSessionEventToSession:612(2178ns) ForwardSessionEventToSession:612(542326ns) ForwardSessionEventToSession:612(2247ns) ForwardSessionEventToSession:612(2296ns)}}}
2025-11-11T17:18:10.117159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518348840283665:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.129781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518344545316343:2107] [node 4] ICP18 handshake (actor [2:7571518348840283665:2120]) is too old
2025-11-11T17:18:10.138905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518349359247906:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:10.013497Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964054:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.013540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518343432964040:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(21104187ns) {HandleHandshakeStatus:329(21100143ns) {DropIncomingHandshake:441(2785ns) DropOutgoingHandshake:456(802ns) SwitchToState:178(671ns) ProcessPendingSessionEvents:495(5220220ns) {ForwardSessionEventToSession:612(3448322ns) ForwardSessionEventToSession:612(4919ns) ForwardSessionEventToSession:612(438400ns) ForwardSessionEventToSession:612(3869ns) ForwardSessionEventToSession:612(2149ns) ForwardSessionEventToSession:612(211848ns) ForwardSessionEventToSession:612(2515ns) ForwardSessionEventToSession:612(1081709ns) ForwardSessionEventToSession:612(9535ns) ForwardSessionEventToSession:612(2226ns)}}}
2025-11-11T17:18:10.013545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518343432964054:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.013952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518343432964041:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.020676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:10.020746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518343432964041:2107] [node 4] ICP22 created new session: [1:7571518347727931365:2107]
2025-11-11T17:18:10.020797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518347727931365:2107] [node 4] ICS09 handshake done sender: [1:7571518343432964047:2112] self: [1:7571518343432964046:0] peer: [4:7571518343391530864:0] socket: 50 qp: -1
2025-11-11T17:18:10.020830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518347727931365:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:10.020865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:10.027864Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518343432964053:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:10.035726Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518343432964054:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:10.045291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518343432964039:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.062296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:10.063330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518343432964039:2108] [node 5] ICP22 created new session: [1:7571518347727931368:2108]
2025-11-11T17:18:10.066137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518347727931368:2108] [node 5] ICS09 handshake done sender: [1:7571518343432964053:2113] self: [1:7571518343432964052:0] peer: [5:7571518343471714879:0] socket: 71 qp: -1
2025-11-11T17:18:10.068698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518343432963977:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.068704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518347727931368:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:10.069222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:10.069264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518343432963977:2049] [node 3] ICP22 created new session: [1:7571518347727931370:2049]
2025-11-11T17:18:10.069611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:10.079769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518347727931370:2049] [node 3] ICS09 handshake done sender: [1:7571518343432964054:2114] self: [1:7571518343432964051:0] peer: [3:7571518343939704344:0] socket: 90 qp: -1
2025-11-11T17:18:10.088013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518347727931370:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:10.089514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:10.133707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518343432963977:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(67416407ns) {HandleHandshakeStatus:329(67412866ns) {DropIncomingHandshake:441(13355ns) DropOutgoingHandshake:456(447ns) SwitchToState:178(538ns) ProcessPendingSessionEvents:495(11265983ns) {ForwardSessionEventToSession:612(6038927ns) ForwardSessionEventToSession:612(5420ns) ForwardSessionEventToSession:612(1311777ns) ForwardSessionEventToSession:612(2277528ns) ForwardSessionEventToSession:612(1588886ns) ForwardSessionEventToSession:612(5672ns) ForwardSessionEventToSession:612(2664ns) ForwardSessionEventToSession:612(4674ns) ForwardSessionEventToSession:612(2327ns) ForwardSessionEventToSession:612(2327ns)}}}
2025-11-11T17:18:10.137777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518343432964039:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(93436307ns) {HandleHandshakeStatus:329(93433493ns) {DropIncomingHandshake:441(12153ns) DropOutgoingHandshake:456(1189ns) SwitchToState:178(503ns) ProcessPendingSessionEvents:495(27309169ns) {ForwardSessionEventToSession:612(5838915ns) ForwardSessionEventToSession:612(5491421ns) ForwardSessionEventToSession:612(2467ns) ForwardSessionEventToSession:612(1692ns) ForwardSessionEventToSession:612(5164070ns) ForwardSessionEventToSession:612(874399ns) ForwardSessionEventToSession:612(3637287ns) ForwardSessionEventToSession:612(3025ns) ForwardSessionEventToSession:612(1958ns) ForwardSessionEventToSession:612(2943933ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.138972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:10.139329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518345064280582:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.139362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:10.139394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518345064280582:2108] [node 4] ICP22 created new session: [6:7571518349359247908:2108]
2025-11-11T17:18:10.139733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518349359247905:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.143481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518345064280520:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.182116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518349359247910:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:10.187113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518347766682177:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
2025-11-11T17:18:10.151816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:10.182312Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518343939704338:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:10.186205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518343939704259:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(68044943ns) {HandleHandshakeStatus:329(68042136ns) {DropIncomingHandshake:441(1377ns) DropOutgoingHandshake:456(6557ns) SwitchToState:178(665ns) ProcessPendingSessionEvents:495(18698307ns) {ForwardSessionEventToSession:612(979508ns) ForwardSessionEventToSession:612(3115ns) ForwardSessionEventToSession:612(2521ns) ForwardSessionEventToSession:612(1964416ns) ForwardSessionEventToSession:612(2910ns) ForwardSessionEventToSession:612(2088ns) ForwardSessionEventToSession:612(5876564ns) ForwardSessionEventToSession:612(1037103ns) ForwardSessionEventToSession:612(8805421ns) ForwardSessionEventToSession:612(6375ns)}}}
2025-11-11T17:18:10.151857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518343391530846:2109] [node 6] ICP22 created new session: [4:7571518347686498170:2109]
2025-11-11T17:18:10.151908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518347686498170:2109] [node 6] ICS09 handshake done sender: [4:7571518343391530858:2116] self: [4:7571518343391530857:0] peer: [6:7571518345064280603:0] socket: 81 qp: -1
2025-11-11T17:18:10.151943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518347686498170:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:10.151971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:10.178002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316362:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.209396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518343939704329:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.209480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:10.187624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518349359247908:2108] [node 4] ICS09 handshake done sender: [6:7571518345064280599:2117] self: [6:7571518345064280603:0] peer: [4:7571518343391530857:0] socket: 83 qp: -1
2025-11-11T17:18:10.209551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518343939704329:2109] [node 6] ICP22 created new session: [3:7571518348234671654:2109]
2025-11-11T17:18:10.217712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518348234671654:2109] [node 6] ICS09 handshake done sender: [3:7571518343939704338:2115] self: [3:7571518343939704337:0] peer: [6:7571518349359247909:0] socket: 65 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.222104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518348234671654:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:10.209298Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518343471714870:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:10.187709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518349359247908:2108] [node 4] ICS10 traffic start
2025-11-11T17:18:10.187792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:10.190293Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518349359247905:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:10.210069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518345064280520:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.210121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:10.210157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518345064280520:2050] [node 3] ICP22 created new session: [6:7571518349359247912:2050]
2025-11-11T17:18:10.210232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518349359247912:2050] [node 3] ICS09 handshake done sender: [6:7571518349359247905:2122] self: [6:7571518349359247909:0] peer: [3:7571518343939704337:0] socket: 51 qp: -1
2025-11-11T17:18:10.210273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518349359247912:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:10.210313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:10.230256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518345064280582:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(90677092ns) {HandleHandshakeStatus:329(90675583ns) {DropIncomingHandshake:441(447ns) DropOutgoingHandshake:456(3801ns) SwitchToState:178(270ns) ProcessPendingSessionEvents:495(134519ns) {ForwardSessionEventToSession:612(67165ns) ForwardSessionEventToSession:612(2584ns) ForwardSessionEventToSession:612(4462ns) ForwardSessionEventToSession:612(33232ns) ForwardSessionEventToSession:612(3226ns) ForwardSessionEventToSession:612(4595ns) ForwardSessionEventToSession:612(2069ns) ForwardSessionEventToSession:612(1994ns) ForwardSessionEventToSession:612(4516ns) ForwardSessionEventToSession:612(1897ns)}}}
2025-11-11T17:18:10.226570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518347766682178:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.239806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518347766682179:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
2025-11-11T17:18:10.231074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:10.225912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518343391530846:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(49265870ns) {HandleHandshakeStatus:329(49262897ns) {DropIncomingHandshake:441(18890ns) DropOutgoingHandshake:456(659ns) SwitchToState:178(636ns) ProcessPendingSessionEvents:495(48874729ns) {ForwardSessionEventToSession:612(6676684ns) ForwardSessionEventToSession:612(6860135ns) ForwardSessionEventToSession:612(12603ns) ForwardSessionEventToSession:612(7215927ns) ForwardSessionEventToSession:612(3223ns) ForwardSessionEventToSession:612(16889153ns) ForwardSessionEventToSession:612(8656ns) ForwardSessionEventToSession:612(2660ns) ForwardSessionEventToSession:612(733032ns) ForwardSessionEventToSession:612(10439398ns)}}}
2025-11-11T17:18:10.253070Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518347766682179:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.253154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518343471714791:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.253249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:10.253300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518343471714791:2048] [node 1] ICP22 created new session: [5:7571518347766682180:2048]
2025-11-11T17:18:10.253346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518347766682180:2048] [node 1] ICS09 handshake done sender: [5:7571518343471714870:2120] self: [5:7571518343471714879:0] peer: [1:7571518343432964052:0] socket: 75 qp: -1
2025-11-11T17:18:10.253379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518347766682180:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:10.253417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:10.254310Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518343471714868:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:10.258153Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:10.249810Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518343939704341:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.264817Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518343391530855:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.291414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518347686498169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
2025-11-11T17:18:10.291585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518343391530784:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518343391530855:2114] held: no
2025-11-11T17:18:10.291671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518343391530784:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.291716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:10.291752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518343391530784:2049] [node 2] ICP22 created new session: [4:7571518347686498172:2049]
2025-11-11T17:18:10.291792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518347686498172:2049] [node 2] ICS09 handshake done sender: [4:7571518343391530855:2114] self: [4:7571518343391530865:0] peer: [2:7571518344545316350:0] socket: 70 qp: -1
2025-11-11T17:18:10.291820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518347686498172:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:10.291915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:10.292523Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518343391530860:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.292547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518343391530860:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.293950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518347686498174:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
2025-11-11T17:18:10.294253Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518343432964062:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.294415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518343432964039:2108] [node 5] ICP18 handshake (actor [1:7571518343432964062:2117]) is too old
2025-11-11T17:18:10.259457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518345064280520:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(49170626ns) {HandleHandshakeStatus:329(49165019ns) {DropIncomingHandshake:441(459ns) DropOutgoingHandshake:456(5038ns) SwitchToState:178(373ns) ProcessPendingSessionEvents:495(28264651ns) {ForwardSessionEventToSession:612(8546655ns) ForwardSessionEventToSession:612(892456ns) ForwardSessionEventToSession:612(8507134ns) ForwardSessionEventToSession:612(10029ns) ForwardSessionEventToSession:612(1384076ns) ForwardSessionEventToSession:612(3996ns) ForwardSessionEventToSession:612(2243ns) ForwardSessionEventToSession:612(7355735ns) ForwardSessionEventToSession:612(2592ns) ForwardSessionEventToSession:612(1536325ns)}}}
2025-11-11T17:18:10.271349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518343471714793:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.278099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518343939704329:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(68394961ns) {HandleHandshakeStatus:329(68389100ns) {DropIncomingHandshake:441(15216ns) DropOutgoingHandshake:456(645ns) SwitchToState:178(619ns) ProcessPendingSessionEvents:495(12431154ns) {ForwardSessionEventToSession:612(12373352ns) ForwardSessionEventToSession:612(14466ns) ForwardSessionEventToSession:612(2029ns) ForwardSessionEventToSession:612(1993ns) ForwardSessionEventToSession:612(12705ns) ForwardSessionEventToSession:612(1503ns) ForwardSessionEventToSession:612(1436ns) ForwardSessionEventToSession:612(4179ns) ForwardSessionEventToSession:612(1708ns) ForwardSessionEventToSession:612(4980ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.271409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:10.307693Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518343391530860:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:10.302200Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518344545316360:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.271449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518343471714793:2050] [node 3] ICP22 created new session: [5:7571518347766682182:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.271491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518347766682182:2050] [node 3] ICS09 handshake done sender: [5:7571518343471714868:2118] self: [5:7571518343471714880:0] peer: [3:7571518343939704330:0] socket: 73 qp: -1
2025-11-11T17:18:10.271525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518347766682182:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:10.271622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:10.281970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518343471714877:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.282077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518343471714792:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.282996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518343471714859:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.307367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518347766682187:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
2025-11-11T17:18:10.318297Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518343471714877:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:10.325984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518343471714859:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.326065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:10.326110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518343471714859:2108] [node 4] ICP22 created new session: [5:7571518347766682188:2108]
2025-11-11T17:18:10.326178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518347766682188:2108] [node 4] ICS09 handshake done sender: [5:7571518343471714877:2123] self: [5:7571518347766682185:0] peer: [4:7571518343391530859:0] socket: 87 qp: -1
2025-11-11T17:18:10.326212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518347766682188:2108] [node 4] ICS10 traffic start
2025-11-11T17:18:10.326249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:10.302254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518344545316360:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.317266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518343391530847:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.317305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:10.317350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518343391530847:2108] [node 5] ICP22 created new session: [4:7571518347686498176:2108]
2025-11-11T17:18:10.317389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518347686498176:2108] [node 5] ICS09 handshake done sender: [4:7571518343391530860:2117] self: [4:7571518343391530859:0] peer: [5:7571518347766682185:0] socket: 82 qp: -1
2025-11-11T17:18:10.317427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518347686498176:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:10.317503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:10.322071Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518347686498169:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.328290Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518347686498174:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.328877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518343391530785:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.328933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518343391530847:2108] [node 5] ICP18 handshake (actor [4:7571518347686498174:2121]) is too old
2025-11-11T17:18:10.341822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518347686498179:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:102
2025-11-11T17:18:10.342331Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:10.343506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518343391530785:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518347686498169:2120] held: no
2025-11-11T17:18:10.343636Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518347686498169:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:10.343902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518343391530785:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.343969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:10.344003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518343391530785:2050] [node 3] ICP22 created new session: [4:7571518347686498180:2050]
2025-11-11T17:18:10.344042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518347686498180:2050] [node 3] ICS09 handshake done sender: [4:7571518347686498169:2120] self: [4:7571518347686498178:0] peer: [3:7571518343939704335:0] socket: 111 qp: -1
2025-11-11T17:18:10.344072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518347686498180:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:10.344101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.329326Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518343939704336:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.329360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518343939704336:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:10.331113Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518343939704336:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:10.331404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518343939704323:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.331450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:10.331496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518343939704323:2107] [node 4] ICP22 created new session: [3:7571518348234671658:2107]
2025-11-11T17:18:10.331536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518348234671658:2107] [node 4] ICS09 handshake done sender: [3:7571518343939704336:2114] self: [3:7571518343939704335:0] peer: [4:7571518347686498178:0] socket: 108 qp: -1
2025-11-11T17:18:10.331564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518348234671658:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:10.331610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:10.349909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518347766682190:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55671
2025-11-11T17:18:10.354198Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518347766682179:2126] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:10.354462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518343471714792:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.354523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:10.354569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518343471714792:2049] [node 2] ICP22 created new session: [5:7571518347766682191:2049]
2025-11-11T17:18:10.354613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518347766682191:2049] [node 2] ICS09 handshake done sender: [5:7571518347766682179:2126] self: [5:7571518347766682184:0] peer: [2:7571518344545316358:0] socket: 64 qp: -1
2025-11-11T17:18:10.374303Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518345064280598:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.374470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518345064280579:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:10.354646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518347766682191:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:10.354716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:10.351228Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518344545316360:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:10.370432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518344545316342:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.370509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:10.370558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518344545316342:2108] [node 5] ICP22 created new session: [2:7571518348840283677:2108]
2025-11-11T17:18:10.370600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518348840283677:2108] [node 5] ICS09 handshake done sender: [2:7571518344545316360:2117] self: [2:7571518344545316358:0] peer: [5:7571518347766682184:0] socket: 59 qp: -1
2025-11-11T17:18:10.370633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518348840283677:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:10.370698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:10.377902Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518343471714867:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:10.377936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518343471714867:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:10.379220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518349359247915:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-11-11T17:18:10.381878Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518343471714867:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:10.382293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518343471714860:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:10.382354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:10.382397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518343471714860:2110] [node 6] ICP22 created new session: [5:7571518347766682194:2110]
2025-11-11T17:18:10.382437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518347766682194:2110] [node 6] ICS09 handshake done sender: [5:7571518343471714867:2117] self: [5:7571518343471714866:0] peer: [6:7571518349359247914:0] socket: 78 qp: -1
2025-11-11T17:18:10.382466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518347766682194:2110] [node 6] ICS10 traffic start
2025-11-11T17:18:10.382544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:10.385892Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:10.386719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518345064280579:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518345064280598:2116] held: no
2025-11-11T17:18:10.387296Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518345064280598:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:10.387555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518345064280579:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:10.387593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:10.387630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518345064280579:2104] [node 5] ICP22 created new session: [6:7571518349359247916:2104]
2025-11-11T17:18:10.387677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518349359247916:2104] [node 5] ICS09 handshake done sender: [6:7571518345064280598:2116] self: [6:7571518349359247914:0] peer: [5:7571518343471714866:0] socket: 79 qp: -1
2025-11-11T17:18:10.387733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518349359247916:2104] [node 5] ICS10 traffic start
2025-11-11T17:18:10.387812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:10.428846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:10.429103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518348840283664:2048] [node 1] ICS07 socket disconnect 56 reason# EndOfStream
2025-11-11T17:18:10.429159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518348840283664:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.429233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:10.429270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518348840283664:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:10.433336Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:18:10.433693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:10.433601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518344545316278:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:18:10.433654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518348840283664:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:10.435222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518348234671651:2048] [node 1] ICS07 socket disconnect 91 reason# EndOfStream
2025-11-11T17:18:10.435254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518348234671651:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.435390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:10.435464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518348234671651:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:10.436039Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address
2025-11-11T17:18:10.436283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518343939704259:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:10.436376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518348234671651:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:10.436409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518343939704259:2048] [node 1] ICP30 unregister session Session# [3:7571518348234671651:2048] VirtualId# [3:7571518343939704344:0]
2025-11-11T17:18:10.436453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:10.436476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518343939704259:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:10.436490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:10.438261Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address
2025-11-11T17:18:10.438897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518344545316278:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:10.440079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518348840283664:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:10.445733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:10.441713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:10.445768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:10.446387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518343939704259:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:10.446122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518347766682180:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-11-11T17:18:10.446228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518343391530868:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-11-11T17:18:10.446161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518343471714791:2048] [node 1] ICP30 unregister session Session# [5:7571518347766682180:2048] VirtualId# [5:7571518343471714879:0]
2025-11-11T17:18:10.446260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518343391530783:2048] [node 1] ICP30 unregister session Session# [4:7571518343391530868:2048] VirtualId# [4:7571518343391530864:0]
2025-11-11T17:18:10.446199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518347766682180:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.446289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518343391530868:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.446374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:10.446322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:10.446377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:10.446412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:10.440131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518344545316278:2048] [node 1] ICP30 unregister session Session# [2:7571518348840283664:2048] VirtualId# [2:7571518344545316355:0]
2025-11-11T17:18:10.440210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:10.440243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518344545316278:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:10.440379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:10.448167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518345064280601:2048] [node 1] ICS01 socket: 55 reason# EndOfStream
2025-11-11T17:18:10.461149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518345064280518:2048] [node 1] ICP30 unregister session Session# [6:7571518345064280601:2048] VirtualId# [6:7571518345064280597:0]
2025-11-11T17:18:10.461199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518345064280601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.461318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:10.461367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:10.502623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:10.502887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518348234671648:2049] [node 2] ICS01 socket: 62 reason# EndOfStream
2025-11-11T17:18:10.502932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518343939704260:2049] [node 2] ICP30 unregister session Session# [3:7571518348234671648:2049] VirtualId# [3:7571518343939704340:0]
2025-11-11T17:18:10.502967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518348234671648:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.503087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:10.503142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:10.509829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:10.510059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518349359247906:2049] [node 2] ICS01 socket: 105 reason# EndOfStream
2025-11-11T17:18:10.510102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518345064280519:2049] [node 2] ICP30 unregister session Session# [6:7571518349359247906:2049] VirtualId# [6:7571518349359247903:0]
2025-11-11T17:18:10.510142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518349359247906:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.510255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:10.510298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:10.510338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:10.510506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518347766682191:2049] [node 2] ICS01 socket: 64 reason# EndOfStream
2025-11-11T17:18:10.510583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518343471714792:2049] [node 2] ICP30 unregister session Session# [5:7571518347766682191:2049] VirtualId# [5:7571518347766682184:0]
2025-11-11T17:18:10.510604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518347766682191:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.510668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:10.510701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:10.509781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:10.510023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518347686498172:2049] [node 2] ICS01 socket: 70 reason# EndOfStream
2025-11-11T17:18:10.510054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518343391530784:2049] [node 2] ICP30 unregister session Session# [4:7571518347686498172:2049] VirtualId# [4:7571518343391530865:0]
2025-11-11T17:18:10.510107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518347686498172:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.510201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:10.510244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:10.548239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:10.548480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518349359247912:2050] [node 3] ICS01 socket: 51 reason# EndOfStream
2025-11-11T17:18:10.548542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518345064280520:2050] [node 3] ICP30 unregister session Session# [6:7571518349359247912:2050] VirtualId# [6:7571518349359247909:0]
2025-11-11T17:18:10.548574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518349359247912:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.548684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:10.548736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:10.550256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:10.550418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518347766682182:2050] [node 3] ICS07 socket disconnect 73 reason# EndOfStream
2025-11-11T17:18:10.550445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518347766682182:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.550492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:10.550532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518347766682182:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:10.549809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:10.549999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518347686498180:2050] [node 3] ICS01 socket: 111 reason# EndOfStream
2025-11-11T17:18:10.550034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518343391530785:2050] [node 3] ICP30 unregister session Session# [4:7571518347686498180:2050] VirtualId# [4:7571518347686498178:0]
2025-11-11T17:18:10.550063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518347686498180:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.550132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:10.550165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:10.551401Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26945) Couldn't connect to any resolved address
2025-11-11T17:18:10.551534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518343471714793:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26945) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:10.551625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518347766682182:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:10.551681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518343471714793:2050] [node 3] ICP30 unregister session Session# [5:7571518347766682182:2050] VirtualId# [5:7571518343471714880:0]
2025-11-11T17:18:10.551745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:10.551783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518343471714793:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:10.551812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:10.566400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518343471714793:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:10.598459Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:10.598660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518349359247908:2108] [node 4] ICS01 socket: 83 reason# EndOfStream
2025-11-11T17:18:10.598699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518345064280582:2108] [node 4] ICP30 unregister session Session# [6:7571518349359247908:2108] VirtualId# [6:7571518345064280603:0]
2025-11-11T17:18:10.598730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518349359247908:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.598900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:10.598939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:10.598910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:10.599112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518347766682188:2108] [node 4] ICS01 socket: 87 reason# EndOfStream
2025-11-11T17:18:10.599368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518343471714859:2108] [node 4] ICP30 unregister session Session# [5:7571518347766682188:2108] VirtualId# [5:7571518347766682185:0]
2025-11-11T17:18:10.599394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518347766682188:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.599463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:10.599513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:10.648640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:10.648858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518349359247916:2104] [node 5] ICS01 socket: 79 reason# EndOfStream
2025-11-11T17:18:10.648888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518345064280579:2104] [node 5] ICP30 unregister session Session# [6:7571518349359247916:2104] VirtualId# [6:7571518349359247914:0]
2025-11-11T17:18:10.648918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518349359247916:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:10.649014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:10.649063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:10.728371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518350009844040:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:10.728419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518350009844041:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:10.751416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518347902903629:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:10.751545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518347902903630:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:10.771625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518347129617105:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:10.771634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518347129617104:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:10.795473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518350003219463:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:10.795523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518350003219464:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:10.795802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518350003219462:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:10.823223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518348097492830:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:10.823479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518348097492831:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:10.823819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518348097492832:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:10.849040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518347990147159:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:10.849102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518347990147161:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:10.849160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518347990147160:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:10.972626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518347990147220:2105] [node 5] ICP01 ready to work
2025-11-11T17:18:11.043589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518354304811420:2127] [node 5] ICP01 ready to work
2025-11-11T17:18:11.084046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518354298186828:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:11.099738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518350009844041:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(121140854ns) {Configure:104(121134875ns) {ProcessConfigured:135(94222ns) {StartInitialHandshake:157(91731ns) {DropHandshakes:469(6003ns) {DropIncomingHandshake:441(868ns) DropOutgoingHandshake:456(644ns)} GenerateSessionVirtualId:988(2123ns) PrepareNewSessionHandshake:476(1955ns) {SwitchToState:178(621ns)}}}}}
2025-11-11T17:18:11.101755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518352197870989:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.113980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518352197870990:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:11.116676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518352197870991:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43046
2025-11-11T17:18:11.105100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518351424584474:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.134213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518351424584475:2108] [node 4] ICP01 ready to work
2025-11-11T17:18:11.137088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518351424584476:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58905
2025-11-11T17:18:11.133822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518352285114522:2109] [node 4] ICP01 ready to work
2025-11-11T17:18:11.146626Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197870991:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.170010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518351424584483:2107] [node 5] ICP01 ready to work
2025-11-11T17:18:11.137971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518354304811428:2107] [node 4] ICP01 ready to work
2025-11-11T17:18:11.149304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518352392460189:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.199080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518354304811429:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-11-11T17:18:11.199099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518354304811430:2119] [node 6] ICP01 ready to work
2025-11-11T17:18:11.206919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518354304811428:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(56527512ns) {Configure:104(56520254ns) {ProcessConfigured:135(81926ns) {StartInitialHandshake:157(79518ns) {DropHandshakes:469(3865ns) {DropIncomingHandshake:441(724ns) DropOutgoingHandshake:456(733ns)} GenerateSessionVirtualId:988(1059ns) PrepareNewSessionHandshake:476(2236ns) {SwitchToState:178(885ns)}}}}}
2025-11-11T17:18:11.172326Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518351424584478:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.172360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518351424584478:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.173390Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518351424584478:2122] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:11.194702Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518351424584476:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.194794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518351424584474:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.194840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:11.194885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518351424584474:2112] [node 6] ICP22 created new session: [3:7571518351424584489:2112]
2025-11-11T17:18:11.194933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518351424584489:2112] [node 6] ICS09 handshake done sender: [3:7571518351424584478:2122] self: [3:7571518351424584477:0] peer: [6:7571518352285114528:0] socket: 61 qp: -1
2025-11-11T17:18:11.194974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518351424584489:2112] [node 6] ICS10 traffic start
2025-11-11T17:18:11.195005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:11.196261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518351424584488:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58905
2025-11-11T17:18:11.196354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518351424584491:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58905
2025-11-11T17:18:11.199249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518352197870998:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.147296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518347990147161:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(137705152ns) {Configure:104(137700089ns) {ProcessConfigured:135(4997469ns) {StartInitialHandshake:157(4994456ns) {DropHandshakes:469(2003ns) {DropIncomingHandshake:441(487ns) DropOutgoingHandshake:456(313ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:476(3666ns) {SwitchToState:178(916ns)}}}}}
2025-11-11T17:18:11.163910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114525:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.163972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114526:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.170748Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518352285114526:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.171090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518347990147161:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.220036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518347990147159:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(118150798ns) {Configure:104(118142302ns) {ProcessConfigured:135(61721ns) {StartInitialHandshake:157(59041ns) {DropHandshakes:469(2425ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(516ns)} GenerateSessionVirtualId:988(899ns) PrepareNewSessionHandshake:476(1463ns) {SwitchToState:178(405ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.293974Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811426:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.294054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518354304811426:2131] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.198783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518347129617104:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(120062339ns) {Configure:104(119480727ns) {ProcessConfigured:135(87027ns) {StartInitialHandshake:157(85103ns) {DropHandshakes:469(3173ns) {DropIncomingHandshake:441(604ns) DropOutgoingHandshake:456(498ns)} GenerateSessionVirtualId:988(1166ns) PrepareNewSessionHandshake:476(2870ns) {SwitchToState:178(838ns)}}}}}
2025-11-11T17:18:11.187701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518350003219463:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(197786537ns) {Configure:104(197780888ns) {ProcessConfigured:135(58203ns) {StartInitialHandshake:157(56663ns) {DropHandshakes:469(2632ns) {DropIncomingHandshake:441(510ns) DropOutgoingHandshake:456(488ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:476(1623ns) {SwitchToState:178(510ns)}}}}}
2025-11-11T17:18:11.315648Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518352285114525:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.284138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518352197870999:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43046
2025-11-11T17:18:11.325411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518347990147160:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(226783377ns) {Configure:104(226773730ns) {ProcessConfigured:135(77922ns) {StartInitialHandshake:157(75766ns) {DropHandshakes:469(3751ns) {DropIncomingHandshake:441(897ns) DropOutgoingHandshake:456(690ns)} GenerateSessionVirtualId:988(1051ns) PrepareNewSessionHandshake:476(2250ns) {SwitchToState:178(827ns)}}}}}
2025-11-11T17:18:11.296112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518354304811433:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-11-11T17:18:11.301682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811429:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.314647Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811433:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.188655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186835:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.221884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518347129617104:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.335651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518348097492830:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(298688354ns) {Configure:104(298683735ns) {ProcessConfigured:135(52596ns) {StartInitialHandshake:157(50857ns) {DropHandshakes:469(3368ns) {DropIncomingHandshake:441(764ns) DropOutgoingHandshake:456(458ns)} GenerateSessionVirtualId:988(885ns) PrepareNewSessionHandshake:476(2040ns) {SwitchToState:178(624ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.326535Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518354304811426:2131] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:11.337679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518347990147160:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.333985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197870993:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.188775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186836:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.246000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518347129617104:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22477799ns) {IncomingHandshake:261(22475278ns) {DropIncomingHandshake:441(354ns) PrepareNewSessionHandshake:476(10236937ns) {SwitchToState:178(3761ns)} GenerateSessionVirtualId:988(1268ns) IssueIncomingHandshakeReply:193(4253600ns) {SwitchToState:178(744ns)}}}
2025-11-11T17:18:11.259519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518351424584480:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.259549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518351424584480:2123] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.261894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518351424584480:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:11.274069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518351424584475:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.274117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:11.274165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518351424584475:2108] [node 4] ICP22 created new session: [3:7571518351424584496:2108]
2025-11-11T17:18:11.274226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518351424584496:2108] [node 4] ICS09 handshake done sender: [3:7571518351424584480:2123] self: [3:7571518351424584479:0] peer: [4:7571518354298186840:0] socket: 62 qp: -1
2025-11-11T17:18:11.274260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518351424584496:2108] [node 4] ICS10 traffic start
2025-11-11T17:18:11.274290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:11.285657Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518351424584491:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.318077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518351424584475:2108] [node 4] ICP18 handshake (actor [3:7571518351424584491:2126]) is too old
2025-11-11T17:18:11.326311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [3:7571518351424584475:2108] [node 4] ICP35 event processing took too much time StateWork:293(20290470ns) {IncomingHandshake:261(20286778ns)}
2025-11-11T17:18:11.327749Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518351424584488:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.357932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518351424584474:2112] [node 6] ICP18 handshake (actor [3:7571518351424584488:2125]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.360630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518352392460192:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.334025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518352197870993:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.350234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114529:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.342339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518354304811430:2119] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(142902632ns) {Configure:104(142894531ns) {ProcessConfigured:135(80737ns) {StartInitialHandshake:157(78292ns) {DropHandshakes:469(2528ns) {DropIncomingHandshake:441(490ns) DropOutgoingHandshake:456(416ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:476(1944ns) {SwitchToState:178(562ns)}}}}}
2025-11-11T17:18:11.199711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518354298186835:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.209494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518350003219464:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55194043ns) {Configure:104(55190637ns) {ProcessConfigured:135(15604591ns) {StartInitialHandshake:157(15602209ns) {DropHandshakes:469(3336ns) {DropIncomingHandshake:441(869ns) DropOutgoingHandshake:456(592ns)} GenerateSessionVirtualId:988(938ns) PrepareNewSessionHandshake:476(3632ns) {SwitchToState:178(1469ns)}}}}}
2025-11-11T17:18:11.264493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518350003219464:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.275740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186844:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.277885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518350003219464:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(46412170ns) {IncomingHandshake:261(46409570ns) {DropIncomingHandshake:441(584ns) PrepareNewSessionHandshake:476(2885ns) {SwitchToState:178(1345ns)} GenerateSessionVirtualId:988(738ns) IssueIncomingHandshakeReply:193(38332885ns) {SwitchToState:178(2433ns)}}}
2025-11-11T17:18:11.353800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518354298186841:2108] [node 5] ICP01 ready to work
2025-11-11T17:18:11.354091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518354298186836:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.354211Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:11.360179Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518354298186835:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:11.373643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518354298186828:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(169476503ns) {Configure:104(169465799ns) {ProcessConfigured:135(75689ns) {StartInitialHandshake:157(73482ns) {DropHandshakes:469(3708ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(818ns)} GenerateSessionVirtualId:988(1821ns) PrepareNewSessionHandshake:476(2249ns) {SwitchToState:178(568ns)}}}}}
2025-11-11T17:18:11.378850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518350003219464:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518354298186835:2120] held: no
2025-11-11T17:18:11.385930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518350003219463:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.358546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114536:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.377425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460193:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
2025-11-11T17:18:11.381854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460194:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
2025-11-11T17:18:11.336338Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518352197870993:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:11.376155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518352285114522:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(210004276ns) {Configure:104(209998404ns) {ProcessConfigured:135(90677ns) {StartInitialHandshake:157(75413ns) {DropHandshakes:469(3463ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(501ns)} GenerateSessionVirtualId:988(1163ns) PrepareNewSessionHandshake:476(1899ns) {SwitchToState:178(644ns)}}}}}
2025-11-11T17:18:11.399409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518354304811428:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(17605248ns) {IncomingHandshake:261(36398ns) {DropIncomingHandshake:441(920ns) PrepareNewSessionHandshake:476(1092ns) {SwitchToState:178(497ns)} GenerateSessionVirtualId:988(429ns) IssueIncomingHandshakeReply:193(4258ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.398627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518347129617105:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(293222342ns) {Configure:104(293219457ns) {ProcessConfigured:135(27024920ns) {StartInitialHandshake:157(27023304ns) {DropHandshakes:469(8928296ns) {DropIncomingHandshake:441(550ns) DropOutgoingHandshake:456(1629ns)} GenerateSessionVirtualId:988(1669ns) PrepareNewSessionHandshake:476(2415ns) {SwitchToState:178(1065ns)}}}}}
2025-11-11T17:18:11.382065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460195:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.420822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518350009844041:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.362491Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197870999:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.401867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518351424584498:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58905
2025-11-11T17:18:11.429751Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518351424584476:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:11.440525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518347129617104:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.440631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:11.440688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518347129617104:2048] [node 1] ICP22 created new session: [3:7571518351424584503:2048]
2025-11-11T17:18:11.454387Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518351424584485:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.454416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518351424584485:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.462695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518351424584503:2048] [node 1] ICS09 handshake done sender: [3:7571518351424584476:2121] self: [3:7571518351424584494:0] peer: [1:7571518354304811425:0] socket: 60 qp: -1
2025-11-11T17:18:11.462759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518351424584503:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:11.462810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:11.463287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518347129617104:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22595943ns) {HandleHandshakeStatus:329(22592427ns) {DropIncomingHandshake:441(733ns) DropOutgoingHandshake:456(4930ns) SwitchToState:178(579ns) ProcessPendingSessionEvents:495(100543ns) {ForwardSessionEventToSession:612(61382ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(5226ns) ForwardSessionEventToSession:612(1537ns) ForwardSessionEventToSession:612(1705ns) ForwardSessionEventToSession:612(3938ns) ForwardSessionEventToSession:612(1883ns) ForwardSessionEventToSession:612(3825ns) ForwardSessionEventToSession:612(6946ns) ForwardSessionEventToSession:612(2033ns)}}}
2025-11-11T17:18:11.463489Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518351424584485:2124] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:11.460576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518350003219464:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(26244677ns) {HandleHandshakeStatus:395(26240095ns) {LogHandshakeFail:472(5293954ns) {UpdateErrorStateLog:544(7629ns)} DropOutgoingHandshake:456(3681ns) ProcessPendingSessionEvents:495(7712482ns) {DropSessionEvent:512(12744ns) DropSessionEvent:512(3043ns) DropSessionEvent:512(6593ns) DropSessionEvent:512(3416ns) DropSessionEvent:512(3092ns) DropSessionEvent:512(2343ns) DropSessionEvent:512(3022ns) DropSessionEvent:512(34979ns) DropSessionEvent:512(9405ns) DropSessionEvent:512(3481ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.508443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518351424584483:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.508493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:11.508529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518351424584483:2107] [node 5] ICP22 created new session: [3:7571518351424584507:2107]
2025-11-11T17:18:11.508574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518351424584507:2107] [node 5] ICS09 handshake done sender: [3:7571518351424584485:2124] self: [3:7571518351424584484:0] peer: [5:7571518352392460206:0] socket: 70 qp: -1
2025-11-11T17:18:11.508605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518351424584507:2107] [node 5] ICS10 traffic start
2025-11-11T17:18:11.508647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:11.509519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518351424584506:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58905
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.382261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460196:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
2025-11-11T17:18:11.435164Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.449704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:11.362538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518352197870989:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.390296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518348097492831:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(396126237ns) {Configure:104(393173649ns) {ProcessConfigured:135(53778ns) {StartInitialHandshake:157(51303ns) {DropHandshakes:469(3236ns) {DropIncomingHandshake:441(796ns) DropOutgoingHandshake:456(504ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:476(2029ns) {SwitchToState:178(695ns)}}}}}
2025-11-11T17:18:11.398177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518348097492832:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(231677346ns) {Configure:104(231671515ns) {ProcessConfigured:135(59547ns) {StartInitialHandshake:157(54796ns) {DropHandshakes:469(3429ns) {DropIncomingHandshake:441(833ns) DropOutgoingHandshake:456(485ns)} GenerateSessionVirtualId:988(1159ns) PrepareNewSessionHandshake:476(2493ns) {SwitchToState:178(737ns)}}}}}
2025-11-11T17:18:11.405913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518352392460195:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.421694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518348097492832:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.435349Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518352392460194:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.440732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518352392460196:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.440903Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518352392460193:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.441031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518348097492830:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.441084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518348097492831:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.447112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518352392460189:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(110439442ns) {Configure:104(110432653ns) {ProcessConfigured:135(59645ns) {StartInitialHandshake:157(57758ns) {DropHandshakes:469(6116ns) {DropIncomingHandshake:441(502ns) DropOutgoingHandshake:456(765ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:476(1992ns) {SwitchToState:178(412ns)}}}}}
2025-11-11T17:18:11.474164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460213:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
2025-11-11T17:18:11.474321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460214:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
2025-11-11T17:18:11.478688Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518352392460195:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:11.479037Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518352392460194:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:11.479254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518348097492832:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.479321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:11.479367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518348097492832:2050] [node 3] ICP22 created new session: [5:7571518352392460215:2050]
2025-11-11T17:18:11.479408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518352392460215:2050] [node 3] ICS09 handshake done sender: [5:7571518352392460195:2113] self: [5:7571518352392460206:0] peer: [3:7571518351424584484:0] socket: 89 qp: -1
2025-11-11T17:18:11.479439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518352392460215:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:11.479472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:11.480679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518348097492830:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.480747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:11.480783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518348097492830:2048] [node 1] ICP22 created new session: [5:7571518352392460218:2048]
2025-11-11T17:18:11.480833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518352392460218:2048] [node 1] ICS09 handshake done sender: [5:7571518352392460194:2112] self: [5:7571518352392460207:0] peer: [1:7571518354304811421:0] socket: 88 qp: -1
2025-11-11T17:18:11.480859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518352392460218:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:11.480886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:11.541864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460212:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
2025-11-11T17:18:11.470171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186849:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.481980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518350003219464:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.482025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:11.503171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518350003219464:2050] [node 3] ICP22 created new session: [4:7571518354298186850:2050]
2025-11-11T17:18:11.507959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186852:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.519784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186853:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.519981Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518354298186836:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:11.520734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518354298186850:2050] [node 3] ICS09 handshake done sender: [4:7571518354298186835:2120] self: [4:7571518354298186840:0] peer: [3:7571518351424584479:0] socket: 71 qp: -1
2025-11-11T17:18:11.461666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518350009844041:2049] [node 3] ICP22 created new session: [1:7571518354304811441:2049]
2025-11-11T17:18:11.362612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:11.438204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518347990147161:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518352285114526:2113] held: no
2025-11-11T17:18:11.438333Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518352285114525:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:11.438919Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518352285114526:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:11.439197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518347990147160:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.439270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:11.439313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518347990147160:2049] [node 2] ICP22 created new session: [6:7571518352285114541:2049]
2025-11-11T17:18:11.439357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518352285114541:2049] [node 2] ICS09 handshake done sender: [6:7571518352285114525:2112] self: [6:7571518352285114535:0] peer: [2:7571518352197870992:0] socket: 64 qp: -1
2025-11-11T17:18:11.439389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518352285114541:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:11.439424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:11.440059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518347990147161:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.440118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:11.440176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518347990147161:2050] [node 3] ICP22 created new session: [6:7571518352285114543:2050]
2025-11-11T17:18:11.440242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518352285114543:2050] [node 3] ICS09 handshake done sender: [6:7571518352285114526:2113] self: [6:7571518352285114528:0] peer: [3:7571518351424584477:0] socket: 65 qp: -1
2025-11-11T17:18:11.440270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518352285114543:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:11.440326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:11.455640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.455764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.517699Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518352285114539:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.519085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114546:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.541753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518347990147220:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.557826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518352285114546:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.468196Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518350009844123:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.566784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518351424584506:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.563122Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518352392460198:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.362662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518352197870989:2109] [node 6] ICP22 created new session: [2:7571518352197871006:2109]
2025-11-11T17:18:11.565778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518347990147220:2105] [node 5] ICP35 event processing took too much time PendingConnection:278(20063866ns) {IncomingHandshake:261(20061080ns) {DropIncomingHandshake:441(514ns) PrepareNewSessionHandshake:476(2783ns) {SwitchToState:178(1048ns)} GenerateSessionVirtualId:988(802ns) IssueIncomingHandshakeReply:193(20031620ns) {SwitchToState:178(576ns)}}}
2025-11-11T17:18:11.621698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518347129617105:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.557620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518354298186850:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:11.566635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518352285114540:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.577764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518347990147159:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.582036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114549:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.593752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518352285114522:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(16028915ns)
2025-11-11T17:18:11.614768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518352285114539:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:11.614999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518347990147220:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.615044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:11.615085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518347990147220:2105] [node 5] ICP22 created new session: [6:7571518352285114552:2105]
2025-11-11T17:18:11.615126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518352285114552:2105] [node 5] ICS09 handshake done sender: [6:7571518352285114539:2119] self: [6:7571518352285114547:0] peer: [5:7571518352392460197:0] socket: 100 qp: -1
2025-11-11T17:18:11.615160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518352285114552:2105] [node 5] ICS10 traffic start
2025-11-11T17:18:11.615194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:11.615676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114551:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.637635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518352285114522:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.637859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518352285114522:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(43870042ns) {IncomingHandshake:261(43866193ns) {DropIncomingHandshake:441(615ns) PrepareNewSessionHandshake:476(2143ns) {SwitchToState:178(706ns)} GenerateSessionVirtualId:988(703ns) IssueIncomingHandshakeReply:193(24038333ns) {SwitchToState:178(3589ns)}}}
2025-11-11T17:18:11.468242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518350009844123:2129] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.563161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518352392460198:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.362753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518352197871006:2109] [node 6] ICS09 handshake done sender: [2:7571518352197870993:2112] self: [2:7571518352197870992:0] peer: [6:7571518352285114535:0] socket: 59 qp: -1
2025-11-11T17:18:11.557743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:11.470423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518354304811439:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-11-11T17:18:11.564254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460220:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.362787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518352197871006:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:11.681807Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:11.706091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518352285114554:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37921
2025-11-11T17:18:11.706604Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518352285114540:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:11.706812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518352285114522:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.706864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:11.706905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518352285114522:2109] [node 4] ICP22 created new session: [6:7571518352285114555:2109]
2025-11-11T17:18:11.706943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518352285114555:2109] [node 4] ICS09 handshake done sender: [6:7571518352285114540:2120] self: [6:7571518352285114550:0] peer: [4:7571518354298186842:0] socket: 101 qp: -1
2025-11-11T17:18:11.706977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518352285114555:2109] [node 4] ICS10 traffic start
2025-11-11T17:18:11.707037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:11.709686Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518352285114546:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:11.713654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518347990147159:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518352285114546:2121] held: no
2025-11-11T17:18:11.557993Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518354298186853:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.470665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811422:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.673721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518347129617105:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(43914903ns) {IncomingHandshake:261(43910607ns) {DropIncomingHandshake:441(488ns) PrepareNewSessionHandshake:476(2210ns) {SwitchToState:178(836ns)} GenerateSessionVirtualId:988(837ns) IssueIncomingHandshakeReply:193(24102519ns) {SwitchToState:178(3216ns)}}}
2025-11-11T17:18:11.570191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518352392460196:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:11.558143Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518354298186849:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.362853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.726278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518351424584510:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58905
2025-11-11T17:18:11.558181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518350003219463:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.558232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:11.558367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518350003219463:2049] [node 2] ICP22 created new session: [4:7571518354298186855:2049]
2025-11-11T17:18:11.558419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518354298186855:2049] [node 2] ICS09 handshake done sender: [4:7571518354298186836:2121] self: [4:7571518354298186847:0] peer: [2:7571518352197870994:0] socket: 72 qp: -1
2025-11-11T17:18:11.558456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518354298186855:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:11.558484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:11.578116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186851:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.641879Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518354298186843:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.641917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518354298186843:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.653656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518350003219464:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(157195173ns) {HandleHandshakeStatus:329(135901934ns) {DropIncomingHandshake:441(469ns) DropOutgoingHandshake:456(175ns) SwitchToState:178(358ns) ProcessPendingSessionEvents:495(1306ns)}}
2025-11-11T17:18:11.653777Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518354298186843:2122] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:11.653895Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518354298186851:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.654047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518350003219462:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.693982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518354298186828:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.701650Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518354298186846:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.701699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518354298186846:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.701730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:11.725116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518354298186828:2109] [node 6] ICP22 created new session: [4:7571518354298186862:2109]
2025-11-11T17:18:11.725209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518354298186862:2109] [node 6] ICS09 handshake done sender: [4:7571518354298186843:2122] self: [4:7571518354298186842:0] peer: [6:7571518352285114550:0] socket: 96 qp: -1
2025-11-11T17:18:11.725255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518354298186862:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:11.725350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:11.729905Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518354298186846:2124] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:11.750349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518354298186865:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59967
2025-11-11T17:18:11.761679Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518351424584506:2130] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:11.389864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518347902903629:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(315459862ns) {Configure:104(315455793ns) {ProcessConfigured:135(19857637ns) {StartInitialHandshake:157(19853890ns) {DropHandshakes:469(3042ns) {DropIncomingHandshake:441(751ns) DropOutgoingHandshake:456(450ns)} GenerateSessionVirtualId:988(1010ns) PrepareNewSessionHandshake:476(3574ns) {SwitchToState:178(1103ns)}}}}}
2025-11-11T17:18:11.414158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518352197870989:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(32732460ns) {HandleHandshakeStatus:329(32729208ns) {DropIncomingHandshake:441(756ns) DropOutgoingHandshake:456(736ns) SwitchToState:178(495ns) ProcessPendingSessionEvents:495(32263371ns) {ForwardSessionEventToSession:612(266822ns) ForwardSessionEventToSession:612(12257ns) ForwardSessionEventToSession:612(2690ns) ForwardSessionEventToSession:612(10551503ns) ForwardSessionEventToSession:612(2724ns) ForwardSessionEventToSession:612(8002733ns) ForwardSessionEventToSession:612(4277ns) ForwardSessionEventToSession:612(2335ns) ForwardSessionEventToSession:612(13371126ns) ForwardSessionEventToSession:612(21495ns)}}}
2025-11-11T17:18:11.470686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518354304811422:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.613763Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518352392460198:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:11.414269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518347902903629:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:11.415256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518352197871009:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43046
2025-11-11T17:18:11.472074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518354304811440:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-11-11T17:18:11.633744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518348097492831:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.647850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:11.653767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518352392460220:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.653944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518352392460189:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:11.661717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518348097492831:2049] [node 2] ICP22 created new session: [5:7571518352392460222:2049]
2025-11-11T17:18:11.677959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518352392460192:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.689900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:11.701703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518352392460192:2109] [node 6] ICP22 created new session: [5:7571518352392460224:2109]
2025-11-11T17:18:11.728521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518352392460222:2049] [node 2] ICS09 handshake done sender: [5:7571518352392460196:2114] self: [5:7571518352392460208:0] peer: [2:7571518352197871000:0] socket: 90 qp: -1
2025-11-11T17:18:11.741042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518352392460222:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:11.741971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518352392460224:2109] [node 6] ICS09 handshake done sender: [5:7571518352392460198:2115] self: [5:7571518352392460197:0] peer: [6:7571518352285114547:0] socket: 91 qp: -1
2025-11-11T17:18:11.741982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:11.760361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518352392460224:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:11.765772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:11.759126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518354298186828:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(31729238ns) {HandleHandshakeStatus:329(31725158ns) {DropIncomingHandshake:441(40667ns) DropOutgoingHandshake:456(358ns) SwitchToState:178(757ns) ProcessPendingSessionEvents:495(145624ns) {ForwardSessionEventToSession:612(107224ns) ForwardSessionEventToSession:612(3086ns) ForwardSessionEventToSession:612(4871ns) ForwardSessionEventToSession:612(2467ns) ForwardSessionEventToSession:612(2109ns) ForwardSessionEventToSession:612(4205ns) ForwardSessionEventToSession:612(2383ns) ForwardSessionEventToSession:612(4542ns) ForwardSessionEventToSession:612(4279ns) ForwardSessionEventToSession:612(1769ns)}}}
2025-11-11T17:18:11.769679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518354298186841:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.769828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518354298186851:2126] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:11.770108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518350003219462:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.770158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:11.770223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518350003219462:2048] [node 1] ICP22 created new session: [4:7571518354298186866:2048]
2025-11-11T17:18:11.770285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518354298186866:2048] [node 1] ICS09 handshake done sender: [4:7571518354298186851:2126] self: [4:7571518354298186861:0] peer: [1:7571518354304811431:0] socket: 115 qp: -1
2025-11-11T17:18:11.770320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518354298186866:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:11.770383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:11.785915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:11.785971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518354298186841:2108] [node 5] ICP22 created new session: [4:7571518354298186868:2108]
2025-11-11T17:18:11.786012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518354298186868:2108] [node 5] ICS09 handshake done sender: [4:7571518354298186846:2124] self: [4:7571518354298186845:0] peer: [5:7571518352392460223:0] socket: 95 qp: -1
2025-11-11T17:18:11.786048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518354298186868:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:11.786124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:11.759851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518347990147159:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(39839238ns) {HandleHandshakeStatus:395(39834877ns) {LogHandshakeFail:472(15022ns) {UpdateErrorStateLog:544(5174ns)} DropOutgoingHandshake:456(3382ns) ProcessPendingSessionEvents:495(15985798ns) {DropSessionEvent:512(12922ns) DropSessionEvent:512(3382ns) DropSessionEvent:512(3017ns) DropSessionEvent:512(2376ns) DropSessionEvent:512(7963021ns) DropSessionEvent:512(3043ns) DropSessionEvent:512(3780ns) DropSessionEvent:512(2784ns) DropSessionEvent:512(2991ns) DropSessionEvent:512(7976884ns)}}}
2025-11-11T17:18:11.759922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518347990147159:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.759995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:11.760036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518347990147159:2048] [node 1] ICP22 created new session: [6:7571518352285114557:2048]
2025-11-11T17:18:11.760076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518352285114557:2048] [node 1] ICS09 handshake done sender: [6:7571518352285114546:2121] self: [6:7571518352285114548:0] peer: [1:7571518354304811435:0] socket: 67 qp: -1
2025-11-11T17:18:11.760107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518352285114557:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:11.760153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:11.826547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518352392460225:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25628
2025-11-11T17:18:11.474547Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518354304811422:2130] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:11.415371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518352197871010:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43046
2025-11-11T17:18:11.817699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518347129617105:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.817783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:11.827813Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:11.857690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518347129617105:2049] [node 2] ICP22 created new session: [3:7571518351424584511:2049]
2025-11-11T17:18:11.874026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518351424584511:2049] [node 2] ICS09 handshake done sender: [3:7571518351424584506:2130] self: [3:7571518351424584509:0] peer: [2:7571518352197871013:0] socket: 118 qp: -1
2025-11-11T17:18:11.441853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518347902903630:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(331379216ns) {Configure:104(331374443ns) {ProcessConfigured:135(20019038ns) {StartInitialHandshake:157(20015302ns) {DropHandshakes:469(2385ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(515ns)} GenerateSessionVirtualId:988(770ns) PrepareNewSessionHandshake:476(3487ns) {SwitchToState:178(1568ns)}}}}}
2025-11-11T17:18:11.474832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518354304811420:2127] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.442424Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197871001:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.442447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518352197871001:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.447361Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197870995:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.458785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518352197870995:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.460212Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518352197871001:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:11.485745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197871009:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.487691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518352197870998:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.501789Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197871010:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.501891Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518352197870995:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:11.517816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:11.541686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518352197870998:2108] [node 5] ICP22 created new session: [2:7571518352197871018:2108]
2025-11-11T17:18:11.561972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518352197870990:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.562041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:11.562083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518352197870990:2107] [node 4] ICP22 created new session: [2:7571518352197871020:2107]
2025-11-11T17:18:11.562134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518352197871020:2107] [node 4] ICS09 handshake done sender: [2:7571518352197870995:2113] self: [2:7571518352197870994:0] peer: [4:7571518354298186847:0] socket: 63 qp: -1
2025-11-11T17:18:11.562167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518352197871020:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:11.562199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:11.566732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518352197871018:2108] [node 5] ICS09 handshake done sender: [2:7571518352197871001:2115] self: [2:7571518352197871000:0] peer: [5:7571518352392460208:0] socket: 81 qp: -1
2025-11-11T17:18:11.577705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518352197871018:2108] [node 5] ICS10 traffic start
2025-11-11T17:18:11.590183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518352197871022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43046
2025-11-11T17:18:11.613707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:11.646515Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518352197870991:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:11.666171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518352197870998:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(178218448ns) {HandleHandshakeStatus:329(178215307ns) {DropIncomingHandshake:441(849ns) DropOutgoingHandshake:456(710ns) SwitchToState:178(921ns) ProcessPendingSessionEvents:495(149752ns) {ForwardSessionEventToSession:612(95339ns) ForwardSessionEventToSession:612(6705ns) ForwardSessionEventToSession:612(2134ns) ForwardSessionEventToSession:612(8941ns) ForwardSessionEventToSession:612(2181ns) ForwardSessionEventToSession:612(2368ns) ForwardSessionEventToSession:612(7199ns) ForwardSessionEventToSession:612(2861ns) ForwardSessionEventToSession:612(2518ns) ForwardSessionEventToSession:612(7456ns)}}}
2025-11-11T17:18:11.666269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518352197870998:2108] [node 5] ICP18 handshake (actor [2:7571518352197871009:2117]) is too old
2025-11-11T17:18:11.673791Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518352197871014:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.673827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518352197871014:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.674979Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:11.675894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518347902903629:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518352197870991:2111] held: no
2025-11-11T17:18:11.685820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518347902903629:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.685905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:11.685957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518347902903629:2048] [node 1] ICP22 created new session: [2:7571518352197871025:2048]
2025-11-11T17:18:11.685999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518352197871025:2048] [node 1] ICS09 handshake done sender: [2:7571518352197870991:2111] self: [2:7571518352197871008:0] peer: [1:7571518350009844102:0] socket: 58 qp: -1
2025-11-11T17:18:11.686033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518352197871025:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:11.686105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:11.693787Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518352197871014:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:11.729947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518347902903630:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.745674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:11.765702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518347902903630:2049] [node 3] ICP22 created new session: [2:7571518352197871027:2049]
2025-11-11T17:18:11.765762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518352197871027:2049] [node 3] ICS09 handshake done sender: [2:7571518352197871014:2119] self: [2:7571518352197871013:0] peer: [3:7571518351424584509:0] socket: 66 qp: -1
2025-11-11T17:18:11.765796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518352197871027:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:11.777714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:11.474873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:11.850108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518348097492831:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(200066288ns) {HandleHandshakeStatus:329(200061816ns) {DropIncomingHandshake:441(4127ns) DropOutgoingHandshake:456(10991ns) SwitchToState:178(760ns) ProcessPendingSessionEvents:495(11989933ns) {ForwardSessionEventToSession:612(11941468ns) ForwardSessionEventToSession:612(8238ns) ForwardSessionEventToSession:612(5067ns) ForwardSessionEventToSession:612(3908ns) ForwardSessionEventToSession:612(2076ns) ForwardSessionEventToSession:612(2181ns) ForwardSessionEventToSession:612(4655ns) ForwardSessionEventToSession:612(2119ns) ForwardSessionEventToSession:612(4525ns) ForwardSessionEventToSession:612(2216ns)}}}
2025-11-11T17:18:11.885976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518351424584511:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:11.893758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:11.889708Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16362) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:11.894258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518352392460189:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16362) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518352392460220:2122] held: no
2025-11-11T17:18:11.894371Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518352392460220:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:11.910098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518352392460189:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:11.910172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:11.910234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518352392460189:2108] [node 4] ICP22 created new session: [5:7571518352392460228:2108]
2025-11-11T17:18:11.910299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518352392460228:2108] [node 4] ICS09 handshake done sender: [5:7571518352392460220:2122] self: [5:7571518352392460223:0] peer: [4:7571518354298186845:0] socket: 119 qp: -1
2025-11-11T17:18:11.910337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518352392460228:2108] [node 4] ICS10 traffic start
2025-11-11T17:18:11.910397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:11.938288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518352392460192:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(257214991ns) {HandleHandshakeStatus:329(257211460ns) {DropIncomingHandshake:441(23552ns) DropOutgoingHandshake:456(545ns) SwitchToState:178(577ns) ProcessPendingSessionEvents:495(21150560ns) {ForwardSessionEventToSession:612(21083669ns) ForwardSessionEventToSession:612(8327ns) ForwardSessionEventToSession:612(3316ns) ForwardSessionEventToSession:612(9904ns) ForwardSessionEventToSession:612(2047ns) ForwardSessionEventToSession:612(3105ns) ForwardSessionEventToSession:612(19399ns) ForwardSessionEventToSession:612(2218ns) ForwardSessionEventToSession:612(2532ns) ForwardSessionEventToSession:612(6374ns)}}}
2025-11-11T17:18:11.474909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518354304811420:2127] [node 5] ICP22 created new session: [1:7571518354304811444:2127]
2025-11-11T17:18:11.474967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518354304811444:2127] [node 5] ICS09 handshake done sender: [1:7571518354304811422:2130] self: [1:7571518354304811421:0] peer: [5:7571518352392460207:0] socket: 56 qp: -1
2025-11-11T17:18:11.474999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518354304811444:2127] [node 5] ICS10 traffic start
2025-11-11T17:18:11.475042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:11.489866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518354304811441:2049] [node 3] ICS09 handshake done sender: [1:7571518354304811426:2131] self: [1:7571518354304811425:0] peer: [3:7571518351424584494:0] socket: 57 qp: -1
2025-11-11T17:18:11.501886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518354304811441:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:11.514293Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811440:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.514478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518354304811447:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-11-11T17:18:11.514634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518354304811420:2127] [node 5] ICP18 handshake (actor [1:7571518354304811440:2137]) is too old
2025-11-11T17:18:11.517618Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:11.520806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:11.547440Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811447:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.553925Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518350009844123:2129] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:11.597890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811436:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.597915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518354304811436:2135] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.606079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518350009844041:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(185004563ns) {HandleHandshakeStatus:329(185001048ns) {DropIncomingHandshake:441(421ns) DropOutgoingHandshake:456(436ns) SwitchToState:178(593ns) ProcessPendingSessionEvents:495(28024050ns) {ForwardSessionEventToSession:612(27934522ns) ForwardSessionEventToSession:612(10445ns) ForwardSessionEventToSession:612(2816ns) ForwardSessionEventToSession:612(6908ns) ForwardSessionEventToSession:612(1666ns) ForwardSessionEventToSession:612(1887ns) ForwardSessionEventToSession:612(36696ns) ForwardSessionEventToSession:612(2419ns) ForwardSessionEventToSession:612(2043ns) ForwardSessionEventToSession:612(7339ns)}}}
2025-11-11T17:18:11.606172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518350009844041:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:11.930035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518350003219462:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(159648502ns) {HandleHandshakeStatus:329(159642002ns) {DropIncomingHandshake:441(617ns) DropOutgoingHandshake:456(5003ns) SwitchToState:178(739ns) ProcessPendingSessionEvents:495(26523677ns) {ForwardSessionEventToSession:612(26471909ns) ForwardSessionEventToSession:612(12365ns) ForwardSessionEventToSession:612(7437ns) ForwardSessionEventToSession:612(1782ns) ForwardSessionEventToSession:612(1484ns) ForwardSessionEventToSession:612(4225ns) ForwardSessionEventToSession:612(2254ns) ForwardSessionEventToSession:612(4646ns) ForwardSessionEventToSession:612(1919ns) ForwardSessionEventToSession:612(4382ns)}}}
2025-11-11T17:18:11.641772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518350009844040:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.641805Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518354304811436:2135] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:11.641857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:11.641905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518350009844040:2048] [node 2] ICP22 created new session: [1:7571518354304811452:2048]
2025-11-11T17:18:11.641996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518354304811452:2048] [node 2] ICS09 handshake done sender: [1:7571518350009844123:2129] self: [1:7571518350009844102:0] peer: [2:7571518352197871008:0] socket: 55 qp: -1
2025-11-11T17:18:11.642966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518354304811430:2119] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.643051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:11.643102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518354304811430:2119] [node 6] ICP22 created new session: [1:7571518354304811453:2119]
2025-11-11T17:18:11.643165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518354304811453:2119] [node 6] ICS09 handshake done sender: [1:7571518354304811436:2135] self: [1:7571518354304811435:0] peer: [6:7571518352285114548:0] socket: 113 qp: -1
2025-11-11T17:18:11.643202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518354304811453:2119] [node 6] ICS10 traffic start
2025-11-11T17:18:11.643240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:11.649706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518354304811452:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:11.673926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:11.702286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518350009844040:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(84100350ns) {HandleHandshakeStatus:329(84096772ns) {DropIncomingHandshake:441(20019ns) DropOutgoingHandshake:456(439ns) SwitchToState:178(663ns) ProcessPendingSessionEvents:495(125405ns) {ForwardSessionEventToSession:612(70558ns) ForwardSessionEventToSession:612(6469ns) ForwardSessionEventToSession:612(3199ns) ForwardSessionEventToSession:612(2261ns) ForwardSessionEventToSession:612(12097ns) ForwardSessionEventToSession:612(2373ns) ForwardSessionEventToSession:612(2351ns) ForwardSessionEventToSession:612(4826ns) ForwardSessionEventToSession:612(1963ns) ForwardSessionEventToSession:612(9513ns)}}}
2025-11-11T17:18:11.709724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518354304811432:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:11.709759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518354304811432:2133] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:11.745741Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518354304811432:2133] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:11.805778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518354304811428:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:11.817675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:11.826670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518354304811428:2107] [node 4] ICP22 created new session: [1:7571518354304811457:2107]
2025-11-11T17:18:11.852114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518354304811457:2107] [node 4] ICS09 handshake done sender: [1:7571518354304811432:2133] self: [1:7571518354304811431:0] peer: [4:7571518354298186861:0] socket: 110 qp: -1
2025-11-11T17:18:11.869747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518354304811457:2107] [node 4] ICS10 traffic start
2025-11-11T17:18:11.894566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:11.926055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518354304811428:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(127896772ns) {HandleHandshakeStatus:329(127893465ns) {DropIncomingHandshake:441(35453ns) DropOutgoingHandshake:456(730ns) SwitchToState:178(714ns) ProcessPendingSessionEvents:495(22618208ns) {ForwardSessionEventToSession:612(22552029ns) ForwardSessionEventToSession:612(9791ns) ForwardSessionEventToSession:612(2460ns) ForwardSessionEventToSession:612(12790ns) ForwardSessionEventToSession:612(2638ns) ForwardSessionEventToSession:612(2103ns) ForwardSessionEventToSession:612(8177ns) ForwardSessionEventToSession:612(2722ns) ForwardSessionEventToSession:612(5046ns) ForwardSessionEventToSession:612(2298ns)}}}
2025-11-11T17:18:11.940197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518354298186841:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(170259681ns) {HandleHandshakeStatus:329(170256294ns) {DropIncomingHandshake:441(8119825ns) DropOutgoingHandshake:456(4413ns) SwitchToState:178(784ns) ProcessPendingSessionEvents:495(34266799ns) {ForwardSessionEventToSession:612(25154034ns) ForwardSessionEventToSession:612(9025487ns) ForwardSessionEventToSession:612(15107ns) ForwardSessionEventToSession:612(1847ns) ForwardSessionEventToSession:612(1346ns) ForwardSessionEventToSession:612(41306ns) ForwardSessionEventToSession:612(1754ns) ForwardSessionEventToSession:612(4445ns) ForwardSessionEventToSession:612(1607ns) ForwardSessionEventToSession:612(1593ns)}}}
2025-11-11T17:18:11.940966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518347129617105:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(120293563ns) {HandleHandshakeStatus:329(120286768ns) {DropIncomingHandshake:441(901ns) DropOutgoingHandshake:456(6840ns) SwitchToState:178(525ns) ProcessPendingSessionEvents:495(170240ns) {ForwardSessionEventToSession:612(122491ns) ForwardSessionEventToSession:612(3329ns) ForwardSessionEventToSession:612(5181ns) ForwardSessionEventToSession:612(2086ns) ForwardSessionEventToSession:612(5165ns) ForwardSessionEventToSession:612(2306ns) ForwardSessionEventToSession:612(2919ns) ForwardSessionEventToSession:612(4331ns) ForwardSessionEventToSession:612(4792ns) ForwardSessionEventToSession:612(2219ns)}}}
2025-11-11T17:18:11.941050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518347129617105:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:11.934501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518347902903630:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(204269630ns) {HandleHandshakeStatus:329(204266900ns) {DropIncomingHandshake:441(16578ns) DropOutgoingHandshake:456(731ns) SwitchToState:178(710ns) ProcessPendingSessionEvents:495(30841722ns) {ForwardSessionEventToSession:612(30797457ns) ForwardSessionEventToSession:612(6671ns) ForwardSessionEventToSession:612(2862ns) ForwardSessionEventToSession:612(4567ns) ForwardSessionEventToSession:612(2315ns) ForwardSessionEventToSession:612(6299ns) ForwardSessionEventToSession:612(3622ns) ForwardSessionEventToSession:612(1632ns) ForwardSessionEventToSession:612(3835ns) ForwardSessionEventToSession:612(1764ns)}}}
2025-11-11T17:18:12.017777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:12.018012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518352197871025:2048] [node 1] ICS01 socket: 58 reason# EndOfStream
2025-11-11T17:18:12.018052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518347902903629:2048] [node 1] ICP30 unregister session Session# [2:7571518352197871025:2048] VirtualId# [2:7571518352197871008:0]
2025-11-11T17:18:12.018083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518352197871025:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.018219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:12.018412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:12.030450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:12.029754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:12.030167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518351424584503:2048] [node 1] ICS01 socket: 60 reason# EndOfStream
2025-11-11T17:18:12.030245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518347129617104:2048] [node 1] ICP30 unregister session Session# [3:7571518351424584503:2048] VirtualId# [3:7571518351424584494:0]
2025-11-11T17:18:12.030284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518351424584503:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.030442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:12.030508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:12.041407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:12.041644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518352285114557:2048] [node 1] ICS01 socket: 67 reason# EndOfStream
2025-11-11T17:18:12.041700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518347990147159:2048] [node 1] ICP30 unregister session Session# [6:7571518352285114557:2048] VirtualId# [6:7571518352285114548:0]
2025-11-11T17:18:12.042075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:12.042264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518352392460218:2048] [node 1] ICS07 socket disconnect 88 reason# EndOfStream
2025-11-11T17:18:12.042288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518352392460218:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.042366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:12.042405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518352392460218:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:12.043198Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address
2025-11-11T17:18:12.049696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518348097492830:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:12.049760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518352392460218:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:12.041742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518352285114557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.041861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:12.041927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:12.057713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518348097492830:2048] [node 1] ICP30 unregister session Session# [5:7571518352392460218:2048] VirtualId# [5:7571518352392460207:0]
2025-11-11T17:18:12.057800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:12.057872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518348097492830:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.057889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.039097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518354298186866:2048] [node 1] ICS07 socket disconnect 115 reason# EndOfStream
2025-11-11T17:18:12.039148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518354298186866:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.039284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:12.039328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518354298186866:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:12.058332Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address
2025-11-11T17:18:12.073741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518350003219462:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:12.101679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518354298186866:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:12.102023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518350003219462:2048] [node 1] ICP30 unregister session Session# [4:7571518354298186866:2048] VirtualId# [4:7571518354298186861:0]
2025-11-11T17:18:12.102088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:12.102122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518350003219462:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.102137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.102382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [4:7571518350003219462:2048] [node 1] ICP35 event processing took too much time StateWork:293(43617701ns) {HandleHandshakeStatus:395(43614678ns) {LogHandshakeFail:472(8608ns) {UpdateErrorStateLog:544(2368ns)} DropOutgoingHandshake:456(6650ns) ProcessPendingSessionEvents:495(233ns) UnregisterSession:538(344158ns) {ProcessPendingSessionEvents:495(551ns) SwitchToInitialState:201(924ns) {SwitchToState:178(443ns)}} TransitToErrorState:752(48300ns) {SwitchToState:178(5735ns) ProcessPendingSessionEvents:495(542ns) DropHandshakes:469(1862ns) {DropIncomingHandshake:441(352ns) DropOutgoingHandshake:456(315ns)}}}}
2025-11-11T17:18:12.101613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518348097492830:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:12.117710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518350003219462:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:12.121866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:12.122111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518351424584511:2049] [node 2] ICS07 socket disconnect 118 reason# EndOfStream
2025-11-11T17:18:12.122147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518351424584511:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.122277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:12.122318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518351424584511:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:12.124232Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:18:12.124858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518347129617105:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:18:12.124889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518351424584511:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:12.125780Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address
2025-11-11T17:18:12.126039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518347129617105:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:12.126084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518351424584511:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:12.126116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518347129617105:2049] [node 2] ICP30 unregister session Session# [3:7571518351424584511:2049] VirtualId# [3:7571518351424584509:0]
2025-11-11T17:18:12.126163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:12.126195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518347129617105:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.126207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.133755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:12.133974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518354298186855:2049] [node 2] ICS01 socket: 72 reason# EndOfStream
2025-11-11T17:18:12.134027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518350003219463:2049] [node 2] ICP30 unregister session Session# [4:7571518354298186855:2049] VirtualId# [4:7571518354298186847:0]
2025-11-11T17:18:12.134062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518354298186855:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.134165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:12.134205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:12.141730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:12.141962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518352285114541:2049] [node 2] ICS01 socket: 64 reason# EndOfStream
2025-11-11T17:18:12.142011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518347990147160:2049] [node 2] ICP30 unregister session Session# [6:7571518352285114541:2049] VirtualId# [6:7571518352285114535:0]
2025-11-11T17:18:12.142040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518352285114541:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.142148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:12.141730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:12.141953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518352392460222:2049] [node 2] ICS07 socket disconnect 90 reason# EndOfStream
2025-11-11T17:18:12.141977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518352392460222:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.142078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:12.142120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518352392460222:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:12.148939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:12.137757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518347129617105:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:12.165909Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address
2025-11-11T17:18:12.166244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518348097492831:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:12.177791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518352392460222:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:12.187842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518348097492831:2049] [node 2] ICP30 unregister session Session# [5:7571518352392460222:2049] VirtualId# [5:7571518352392460208:0]
2025-11-11T17:18:12.187933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:12.187978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518348097492831:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.187995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9896) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:12.188283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [5:7571518348097492831:2049] [node 2] ICP35 event processing took too much time StateWork:293(21834337ns) {HandleHandshakeStatus:395(21830146ns) {LogHandshakeFail:472(17661ns) {UpdateErrorStateLog:544(3446ns)} DropOutgoingHandshake:456(420ns) ProcessPendingSessionEvents:495(960ns) UnregisterSession:538(10070894ns) {ProcessPendingSessionEvents:495(14758ns) SwitchToInitialState:201(2253ns) {SwitchToState:178(1535ns)}} TransitToErrorState:752(55879ns) {SwitchToState:178(5667ns) ProcessPendingSessionEvents:495(557ns) DropHandshakes:469(2173ns) {DropIncomingHandshake:441(405ns) DropOutgoingHandshake:456(478ns)}}}}
2025-11-11T17:18:12.197894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518348097492831:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:12.198305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:12.198871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518352392460215:2050] [node 3] ICS01 socket: 89 reason# EndOfStream
2025-11-11T17:18:12.198951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518348097492832:2050] [node 3] ICP30 unregister session Session# [5:7571518352392460215:2050] VirtualId# [5:7571518352392460206:0]
2025-11-11T17:18:12.199020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518352392460215:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.199135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:12.199196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:12.201741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:12.201961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518354298186850:2050] [node 3] ICS01 socket: 71 reason# EndOfStream
2025-11-11T17:18:12.202001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518350003219464:2050] [node 3] ICP30 unregister session Session# [4:7571518354298186850:2050] VirtualId# [4:7571518354298186840:0]
2025-11-11T17:18:12.202029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518354298186850:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.202135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:12.202176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:12.205766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:12.206022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518352285114543:2050] [node 3] ICS01 socket: 65 reason# EndOfStream
2025-11-11T17:18:12.206061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518347990147161:2050] [node 3] ICP30 unregister session Session# [6:7571518352285114543:2050] VirtualId# [6:7571518352285114528:0]
2025-11-11T17:18:12.206095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518352285114543:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.206213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:12.206256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:12.252516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:12.252724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518352392460228:2108] [node 4] ICS01 socket: 119 reason# EndOfStream
2025-11-11T17:18:12.252763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518352392460189:2108] [node 4] ICP30 unregister session Session# [5:7571518352392460228:2108] VirtualId# [5:7571518352392460223:0]
2025-11-11T17:18:12.252793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518352392460228:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.252876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:12.252915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:12.253745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:12.253927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518352285114555:2109] [node 4] ICS01 socket: 101 reason# EndOfStream
2025-11-11T17:18:12.253965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518352285114522:2109] [node 4] ICP30 unregister session Session# [6:7571518352285114555:2109] VirtualId# [6:7571518352285114550:0]
2025-11-11T17:18:12.253988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518352285114555:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.254045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:12.254088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:12.317770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:12.318027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518352285114552:2105] [node 5] ICS01 socket: 100 reason# EndOfStream
2025-11-11T17:18:12.318065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518347990147220:2105] [node 5] ICP30 unregister session Session# [6:7571518352285114552:2105] VirtualId# [6:7571518352285114547:0]
2025-11-11T17:18:12.318098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518352285114552:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:12.318229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:12.318283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:12.438425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518356541196087:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:12.438465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518356541196088:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:12.482003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518356998237810:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:12.482059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518356998237811:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:12.520011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518358276670056:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:12.520019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518358276670057:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:12.549845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518355411151319:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:12.549881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518355411151320:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:12.549899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518355411151321:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:12.597494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518358955329133:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:12.597528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518358955329134:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:12.597549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518358955329135:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:12.891763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518358276670086:2074] [node 5] ICP01 ready to work
2025-11-11T17:18:12.896788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518356998237841:2084] [node 5] ICP01 ready to work
2025-11-11T17:18:12.892084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518358276670087:2079] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.042687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518359706118648:2079] [node 6] ICP01 ready to work
2025-11-11T17:18:13.127693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518359706118661:2090] [node 5] ICP01 ready to work
2025-11-11T17:18:13.066989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518356998237811:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(36948351ns) {Configure:104(17022315ns) {ProcessConfigured:135(72776ns) {StartInitialHandshake:157(70328ns) {DropHandshakes:469(2134ns) {DropIncomingHandshake:441(491ns) DropOutgoingHandshake:456(371ns)} GenerateSessionVirtualId:988(850ns) PrepareNewSessionHandshake:476(2397ns) {SwitchToState:178(993ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.226654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118668:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:12.928628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518358276670087:2079] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(34952680ns) {Configure:104(34949171ns) {ProcessConfigured:135(71189ns) {StartInitialHandshake:157(68976ns) {DropHandshakes:469(2330ns) {DropIncomingHandshake:441(433ns) DropOutgoingHandshake:456(422ns)} GenerateSessionVirtualId:988(714ns) PrepareNewSessionHandshake:476(7488ns) {SwitchToState:178(463ns)}}}}}
2025-11-11T17:18:13.152127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518358955329151:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:13.168820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518363250296453:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:13.226147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296466:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:13.226205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296465:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.253525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296486:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:12.929006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518358276670093:2081] [node 6] ICP01 ready to work
2025-11-11T17:18:13.086094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518361293205154:2081] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.087720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518361293205155:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18198
2025-11-11T17:18:13.233709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518363250296472:2061] [node 6] ICP01 ready to work
2025-11-11T17:18:13.333931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518360836163446:2079] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.393774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518359706118668:2094] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.194031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518362571637423:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4095
2025-11-11T17:18:13.305984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518358276670093:2081] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(376583437ns) {Configure:104(376579260ns) {ProcessConfigured:135(84338ns) {StartInitialHandshake:157(79103ns) {DropHandshakes:469(2703ns) {DropIncomingHandshake:441(530ns) DropOutgoingHandshake:456(386ns)} GenerateSessionVirtualId:988(935ns) PrepareNewSessionHandshake:476(2413ns) {SwitchToState:178(967ns)}}}}}
2025-11-11T17:18:13.381840Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518362571637423:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.057020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518356624159152:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.143723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518356998237841:2084] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(246636107ns) {Configure:104(246631576ns) {ProcessConfigured:135(21769945ns) {StartInitialHandshake:157(21766568ns) {DropHandshakes:469(2834ns) {DropIncomingHandshake:441(803ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:476(5309ns) {SwitchToState:178(2512ns)}}}}}
2025-11-11T17:18:13.397975Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29329) Couldn't connect to any resolved address
2025-11-11T17:18:13.335876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518360836163448:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47161
2025-11-11T17:18:13.337109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518360836163447:2082] [node 5] ICP01 ready to work
2025-11-11T17:18:13.393912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518360836163452:2080] [node 4] ICP01 ready to work
2025-11-11T17:18:13.401817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163448:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.057063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518356624159153:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:13.410677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518358955329151:2065] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.389996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518361293205180:2083] [node 4] ICP01 ready to work
2025-11-11T17:18:13.414826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518363250296466:2081] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.429876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518359706118648:2079] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29329) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:13.057112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518356624159154:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:13.454002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518362571637428:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4095
2025-11-11T17:18:13.458294Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518363250296465:2080] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.057730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518360919126462:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:13.430028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518359706118648:2079] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:13.502208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518360836163447:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(164743870ns) {Configure:104(164737383ns) {ProcessConfigured:135(106235ns) {StartInitialHandshake:157(103777ns) {DropHandshakes:469(6380ns) {DropIncomingHandshake:441(897ns) DropOutgoingHandshake:456(582ns)} GenerateSessionVirtualId:988(1228ns) PrepareNewSessionHandshake:476(2678ns) {SwitchToState:178(808ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.499353Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518361293205157:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.499413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518361293205157:2093] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.515776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518361293205168:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.515799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518361293205168:2102] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.516607Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518361293205168:2102] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:13.516992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518356998237841:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.517055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:13.517116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518356998237841:2084] [node 5] ICP22 created new session: [2:7571518361293205184:2084]
2025-11-11T17:18:13.517161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518361293205184:2084] [node 5] ICS09 handshake done sender: [2:7571518361293205168:2102] self: [2:7571518361293205160:0] peer: [5:7571518363250296510:0] socket: 71 qp: -1
2025-11-11T17:18:13.517192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518361293205184:2084] [node 5] ICS10 traffic start
2025-11-11T17:18:13.517224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:13.529938Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518361293205157:2093] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:13.463597Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518362571637425:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.430047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.246222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518360919126472:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.463619Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518362571637425:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.441684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518359706118648:2079] [node 6] ICP33 wake up from error state
2025-11-11T17:18:13.473719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518358955329135:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(258292708ns) {Configure:104(258281109ns) {ProcessConfigured:135(35020948ns) {StartInitialHandshake:157(35017289ns) {DropHandshakes:469(18729328ns) {DropIncomingHandshake:441(3152ns) DropOutgoingHandshake:456(301ns)} GenerateSessionVirtualId:988(2184ns) PrepareNewSessionHandshake:476(4643ns) {SwitchToState:178(1583ns)}}}}}
2025-11-11T17:18:13.473881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518358955329135:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.395237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518356624159154:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21413168ns) {Configure:104(16809026ns) {ProcessConfigured:135(100753ns) {StartInitialHandshake:157(98865ns) {DropHandshakes:469(5349ns) {DropIncomingHandshake:441(578ns) DropOutgoingHandshake:456(364ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(2534ns) {SwitchToState:178(898ns)}}}}}
2025-11-11T17:18:13.465562Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518362571637425:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.513830Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518359706118663:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.577680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163450:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.414054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126523:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.414301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126524:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.433942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126522:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.457685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518360919126523:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.462295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518360919126462:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(402925025ns) {Configure:104(402919419ns) {ProcessConfigured:135(6905426ns) {StartInitialHandshake:157(6902415ns) {DropHandshakes:469(2937ns) {DropIncomingHandshake:441(665ns) DropOutgoingHandshake:456(553ns)} GenerateSessionVirtualId:988(1407ns) PrepareNewSessionHandshake:476(3825ns) {SwitchToState:178(2162ns)}}}}}
2025-11-11T17:18:13.462442Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518360919126522:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.462506Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518360919126524:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.462735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518356624159154:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.462921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518356624159153:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.465352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126529:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.543791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518356624159152:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(170124569ns) {Configure:104(170117931ns) {ProcessConfigured:135(88947ns) {StartInitialHandshake:157(86072ns) {DropHandshakes:469(3318ns) {DropIncomingHandshake:441(743ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(1254ns) PrepareNewSessionHandshake:476(2355ns) {SwitchToState:178(721ns)}}}}}
2025-11-11T17:18:13.544054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518356624159152:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.558278Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518360919126523:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:13.559927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518361293205182:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18198
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.559424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518358276670057:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(497312381ns) {Configure:104(497309212ns) {ProcessConfigured:135(85626ns) {StartInitialHandshake:157(83300ns) {DropHandshakes:469(2421ns) {DropIncomingHandshake:441(642ns) DropOutgoingHandshake:456(365ns)} GenerateSessionVirtualId:988(1143ns) PrepareNewSessionHandshake:476(2597ns) {SwitchToState:178(1006ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.599935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518361293205154:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.577747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518360836163450:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.513863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518359706118663:2091] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.494936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518363250296453:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(174069676ns) {Configure:104(174060874ns) {ProcessConfigured:135(97104ns) {StartInitialHandshake:157(90576ns) {DropHandshakes:469(3269ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(698ns)} GenerateSessionVirtualId:988(1365ns) PrepareNewSessionHandshake:476(2601ns) {SwitchToState:178(1046ns)}}}}}
2025-11-11T17:18:13.648265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518356624159154:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.614632Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518360836163450:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:13.495205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518363250296453:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.578210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518358276670093:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.539405Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518359706118663:2091] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:13.599986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.651494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126536:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.641249Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163454:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.578287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:13.558751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118689:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.606095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518356998237810:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(533075448ns) {Configure:104(533072507ns) {ProcessConfigured:135(20818507ns) {StartInitialHandshake:157(20815182ns) {DropHandshakes:469(3084ns) {DropIncomingHandshake:441(715ns) DropOutgoingHandshake:456(594ns)} GenerateSessionVirtualId:988(817ns) PrepareNewSessionHandshake:476(2495ns) {SwitchToState:178(161ns)}}}}}
2025-11-11T17:18:13.514847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518358955329134:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(517183567ns) {Configure:104(517174373ns) {ProcessConfigured:135(16620434ns) {StartInitialHandshake:157(16616832ns) {DropHandshakes:469(2097ns) {DropIncomingHandshake:441(478ns) DropOutgoingHandshake:456(371ns)} GenerateSessionVirtualId:988(1101ns) PrepareNewSessionHandshake:476(2389ns) {SwitchToState:178(855ns)}}}}}
2025-11-11T17:18:13.515047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518358955329134:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.545668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296512:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:13.550275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518363250296472:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(204381598ns) {Configure:104(192450345ns) {ProcessConfigured:135(20352889ns) {StartInitialHandshake:157(20349735ns) {DropHandshakes:469(3859ns) {DropIncomingHandshake:441(1152ns) DropOutgoingHandshake:456(705ns)} GenerateSessionVirtualId:988(832ns) PrepareNewSessionHandshake:476(4157ns) {SwitchToState:178(1460ns)}}}}}
2025-11-11T17:18:13.550884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296515:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:13.551268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296516:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:13.585957Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518363250296465:2080] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:13.593817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518363250296516:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.594761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518358955329133:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.599878Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518363250296466:2081] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:13.604198Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518363250296486:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.617991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518358955329134:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.618058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:13.618102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518358955329134:2049] [node 2] ICP22 created new session: [5:7571518363250296518:2049]
2025-11-11T17:18:13.618144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518363250296518:2049] [node 2] ICS09 handshake done sender: [5:7571518363250296466:2081] self: [5:7571518363250296510:0] peer: [2:7571518361293205160:0] socket: 73 qp: -1
2025-11-11T17:18:13.618175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518363250296518:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:13.618209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:13.620076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518363250296453:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.620153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:13.620199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518363250296453:2059] [node 4] ICP22 created new session: [5:7571518363250296520:2059]
2025-11-11T17:18:13.620245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518363250296520:2059] [node 4] ICS09 handshake done sender: [5:7571518363250296465:2080] self: [5:7571518363250296506:0] peer: [4:7571518359706118662:0] socket: 72 qp: -1
2025-11-11T17:18:13.620279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518363250296520:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:13.620337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:13.650892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296524:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:13.679494Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518358955329151:2065] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:13.696985Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:13.698825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518358955329135:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518358955329151:2065] held: no
2025-11-11T17:18:13.707716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518358955329135:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.707767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:13.707816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518358955329135:2050] [node 3] ICP22 created new session: [5:7571518363250296526:2050]
2025-11-11T17:18:13.707876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518363250296526:2050] [node 3] ICS09 handshake done sender: [5:7571518358955329151:2065] self: [5:7571518363250296503:0] peer: [3:7571518358276670088:0] socket: 58 qp: -1
2025-11-11T17:18:13.707906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518363250296526:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:13.707950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:13.714563Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518363250296514:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.714583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518363250296514:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.651935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.614891Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518361293205155:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.609942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518359706118689:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.578358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518358276670093:2081] [node 6] ICP22 created new session: [3:7571518362571637433:2081]
2025-11-11T17:18:13.641275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518360836163454:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.659041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518356624159154:2050] [node 3] ICP22 created new session: [6:7571518360919126535:2050]
2025-11-11T17:18:13.746287Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518363250296514:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.629798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518361293205154:2081] [node 6] ICP22 created new session: [2:7571518361293205190:2081]
2025-11-11T17:18:13.646309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518356541196088:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(584351946ns) {Configure:104(564407520ns) {ProcessConfigured:135(31150362ns) {StartInitialHandshake:157(31146805ns) {DropHandshakes:469(10957492ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(1916ns)} GenerateSessionVirtualId:988(1903ns) PrepareNewSessionHandshake:476(5203ns) {SwitchToState:178(2627ns)}}}}}
2025-11-11T17:18:13.614694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518359706118661:2090] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.798434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518363250296472:2061] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.653810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518361293205190:2081] [node 6] ICS09 handshake done sender: [2:7571518361293205157:2093] self: [2:7571518361293205156:0] peer: [6:7571518360919126528:0] socket: 68 qp: -1
2025-11-11T17:18:13.800764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.810179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518363250296472:2061] [node 6] ICP22 created new session: [5:7571518363250296530:2061]
2025-11-11T17:18:13.810251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518363250296530:2061] [node 6] ICS09 handshake done sender: [5:7571518363250296514:2119] self: [5:7571518363250296513:0] peer: [6:7571518360919126539:0] socket: 86 qp: -1
2025-11-11T17:18:13.810290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518363250296530:2061] [node 6] ICS10 traffic start
2025-11-11T17:18:13.810342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:13.810815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518363250296472:2061] [node 6] ICP35 event processing took too much time PendingConnection:278(20338000ns) {HandleHandshakeStatus:329(20334745ns) {DropIncomingHandshake:441(28530ns) DropOutgoingHandshake:456(773ns) SwitchToState:178(572ns) ProcessPendingSessionEvents:495(146737ns) {ForwardSessionEventToSession:612(57680ns) ForwardSessionEventToSession:612(7959ns) ForwardSessionEventToSession:612(1873ns) ForwardSessionEventToSession:612(32324ns) ForwardSessionEventToSession:612(7784ns) ForwardSessionEventToSession:612(1341ns) ForwardSessionEventToSession:612(4951ns) ForwardSessionEventToSession:612(1344ns) ForwardSessionEventToSession:612(4249ns) ForwardSessionEventToSession:612(18800ns)}}}
2025-11-11T17:18:13.701302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518356541196087:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(564001816ns) {Configure:104(563990635ns) {ProcessConfigured:135(20030962ns) {StartInitialHandshake:157(20026434ns) {DropHandshakes:469(2354ns) {DropIncomingHandshake:441(479ns) DropOutgoingHandshake:456(339ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:476(4412ns) {SwitchToState:178(1959ns)}}}}}
2025-11-11T17:18:13.653863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518361293205190:2081] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.738941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518360836163446:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.815114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518363250296529:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62474
2025-11-11T17:18:13.653944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:13.618083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518355411151321:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(539402498ns) {Configure:104(539397520ns) {ProcessConfigured:135(37849494ns) {StartInitialHandshake:157(37846612ns) {DropHandshakes:469(2450ns) {DropIncomingHandshake:441(602ns) DropOutgoingHandshake:456(110ns)} GenerateSessionVirtualId:988(995ns) PrepareNewSessionHandshake:476(3102ns) {SwitchToState:178(651ns)}}}}}
2025-11-11T17:18:13.578411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518362571637433:2081] [node 6] ICS09 handshake done sender: [3:7571518362571637425:2112] self: [3:7571518362571637424:0] peer: [6:7571518360919126527:0] socket: 77 qp: -1
2025-11-11T17:18:13.654450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518361293205154:2081] [node 6] ICP35 event processing took too much time PendingConnection:278(58911508ns) {HandleHandshakeStatus:329(54328339ns) {DropIncomingHandshake:441(222ns) DropOutgoingHandshake:456(736ns) SwitchToState:178(747ns) ProcessPendingSessionEvents:495(142151ns) {ForwardSessionEventToSession:612(62236ns) ForwardSessionEventToSession:612(5784ns) ForwardSessionEventToSession:612(2636ns) ForwardSessionEventToSession:612(4761ns) ForwardSessionEventToSession:612(2674ns) ForwardSessionEventToSession:612(2879ns) ForwardSessionEventToSession:612(4683ns) ForwardSessionEventToSession:612(3057ns) ForwardSessionEventToSession:612(4862ns) ForwardSessionEventToSession:612(37269ns)}}}
2025-11-11T17:18:13.654587Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:13.666305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518356998237841:2084] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:13.666494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518361293205154:2081] [node 6] ICP18 handshake (actor [2:7571518361293205155:2092]) is too old
2025-11-11T17:18:13.674861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518361293205194:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18198
2025-11-11T17:18:13.679674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518361293205150:2089] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.679709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518361293205150:2089] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.701874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518361293205150:2089] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:13.713952Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518361293205194:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.728244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518356998237811:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.728320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:13.728388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518356998237811:2049] [node 3] ICP22 created new session: [2:7571518361293205198:2049]
2025-11-11T17:18:13.728436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518361293205198:2049] [node 3] ICS09 handshake done sender: [2:7571518361293205150:2089] self: [2:7571518361293205149:0] peer: [3:7571518362571637437:0] socket: 65 qp: -1
2025-11-11T17:18:13.728476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518361293205198:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:13.728511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:13.730330Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518361293205195:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.730350Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518361293205195:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.737880Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518361293205195:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:13.750066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518361293205180:2083] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(252120190ns) {Configure:104(252108732ns) {ProcessConfigured:135(25275462ns) {StartInitialHandshake:157(25271941ns) {DropHandshakes:469(2169ns) {DropIncomingHandshake:441(442ns) DropOutgoingHandshake:456(416ns)} GenerateSessionVirtualId:988(750ns) PrepareNewSessionHandshake:476(3266ns) {SwitchToState:178(974ns)}}}}}
2025-11-11T17:18:13.755567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518361293205180:2083] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.762096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:13.762158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518361293205180:2083] [node 4] ICP22 created new session: [2:7571518361293205202:2083]
2025-11-11T17:18:13.762218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518361293205202:2083] [node 4] ICS09 handshake done sender: [2:7571518361293205195:2118] self: [2:7571518361293205192:0] peer: [4:7571518359706118704:0] socket: 87 qp: -1
2025-11-11T17:18:13.762254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518361293205202:2083] [node 4] ICS10 traffic start
2025-11-11T17:18:13.762293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:13.762812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518361293205191:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18198
2025-11-11T17:18:13.762975Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:13.784726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518356998237811:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:13.814308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518361293205204:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18198
2025-11-11T17:18:13.829809Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518361293205204:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.831243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518356998237810:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.761839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:13.659123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518360919126535:2050] [node 3] ICS09 handshake done sender: [6:7571518360919126523:2114] self: [6:7571518360919126527:0] peer: [3:7571518362571637424:0] socket: 81 qp: -1
2025-11-11T17:18:13.633828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518355411151321:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.578451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518362571637433:2081] [node 6] ICS10 traffic start
2025-11-11T17:18:13.774226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518360836163446:2079] [node 6] ICP22 created new session: [1:7571518360836163465:2079]
2025-11-11T17:18:13.842236Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:13.659157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518360919126535:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:13.637777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.578511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:13.849886Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518363250296516:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:13.778133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518360836163461:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47161
2025-11-11T17:18:13.659209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:13.649860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518359706118661:2090] [node 5] ICP22 created new session: [4:7571518359706118696:2090]
2025-11-11T17:18:13.782673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518360836163467:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47161
2025-11-11T17:18:13.580028Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518362571637428:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.900057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518358955329133:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518363250296516:2121] held: no
2025-11-11T17:18:13.899668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518361293205206:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18198
2025-11-11T17:18:13.782984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518360836163468:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47161
2025-11-11T17:18:13.943800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518358955329133:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(56088426ns) {HandleHandshakeStatus:395(56082776ns) {LogHandshakeFail:472(23926ns) {UpdateErrorStateLog:544(5760ns)} DropOutgoingHandshake:456(3276ns) ProcessPendingSessionEvents:495(9648699ns) {DropSessionEvent:512(15149ns) DropSessionEvent:512(11240ns) DropSessionEvent:512(4961ns) DropSessionEvent:512(9577291ns) DropSessionEvent:512(8892ns) DropSessionEvent:512(11216ns) DropSessionEvent:512(3042ns) DropSessionEvent:512(3564ns) DropSessionEvent:512(2432ns) DropSessionEvent:512(2596ns)}}}
2025-11-11T17:18:13.943882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518358955329133:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.597951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518358276670056:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(699730189ns) {Configure:104(699726747ns) {ProcessConfigured:135(41093384ns) {StartInitialHandshake:157(41089933ns) {DropHandshakes:469(3291ns) {DropIncomingHandshake:441(1095ns) DropOutgoingHandshake:456(615ns)} GenerateSessionVirtualId:988(984ns) PrepareNewSessionHandshake:476(11230ns) {SwitchToState:178(876ns)}}}}}
2025-11-11T17:18:13.655175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518355411151320:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(548146912ns) {Configure:104(548138683ns) {ProcessConfigured:135(5227825ns) {StartInitialHandshake:157(5224280ns) {DropHandshakes:469(2392ns) {DropIncomingHandshake:441(401ns) DropOutgoingHandshake:456(514ns)} GenerateSessionVirtualId:988(948ns) PrepareNewSessionHandshake:476(3379ns) {SwitchToState:178(1516ns)}}}}}
2025-11-11T17:18:13.659656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518356624159154:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(99262263ns) {HandleHandshakeStatus:329(99259379ns) {DropIncomingHandshake:441(2582ns) DropOutgoingHandshake:456(27488ns) SwitchToState:178(414ns) ProcessPendingSessionEvents:495(101800ns) {ForwardSessionEventToSession:612(56084ns) ForwardSessionEventToSession:612(3124ns) ForwardSessionEventToSession:612(2658ns) ForwardSessionEventToSession:612(10712ns) ForwardSessionEventToSession:612(4854ns) ForwardSessionEventToSession:612(4328ns) ForwardSessionEventToSession:612(2083ns) ForwardSessionEventToSession:612(2343ns) ForwardSessionEventToSession:612(4150ns) ForwardSessionEventToSession:612(1686ns)}}}
2025-11-11T17:18:13.790623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518360836163452:2080] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(370879758ns) {Configure:104(370871400ns) {ProcessConfigured:135(32734456ns) {StartInitialHandshake:157(32731138ns) {DropHandshakes:469(3986ns) {DropIncomingHandshake:441(861ns) DropOutgoingHandshake:456(646ns)} GenerateSessionVirtualId:988(1281ns) PrepareNewSessionHandshake:476(2654ns) {SwitchToState:178(808ns)}}}}}
2025-11-11T17:18:13.942899Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518361293205204:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:13.961822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:13.660843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518355411151319:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(703893487ns) {Configure:104(703887353ns) {ProcessConfigured:135(59282756ns) {StartInitialHandshake:157(59279678ns) {DropHandshakes:469(2312ns) {DropIncomingHandshake:441(695ns) DropOutgoingHandshake:456(360ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(4837ns) {SwitchToState:178(1933ns)}}}}}
2025-11-11T17:18:13.627758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518358276670089:2080] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.678067Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5703) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:13.977841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518358955329133:2048] [node 1] ICP22 created new session: [5:7571518363250296532:2048]
2025-11-11T17:18:13.978923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518363250296532:2048] [node 1] ICS09 handshake done sender: [5:7571518363250296516:2121] self: [5:7571518363250296517:0] peer: [1:7571518360836163453:0] socket: 96 qp: -1
2025-11-11T17:18:13.985889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518363250296532:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:13.675045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518359706118696:2090] [node 5] ICS09 handshake done sender: [4:7571518359706118663:2091] self: [4:7571518359706118662:0] peer: [5:7571518363250296506:0] socket: 70 qp: -1
2025-11-11T17:18:13.675112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518359706118696:2090] [node 5] ICS10 traffic start
2025-11-11T17:18:13.675175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:13.675650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518359706118661:2090] [node 5] ICP35 event processing took too much time PendingConnection:278(100736507ns) {HandleHandshakeStatus:329(100730269ns) {DropIncomingHandshake:441(2548ns) DropOutgoingHandshake:456(1562ns) SwitchToState:178(695ns) ProcessPendingSessionEvents:495(92205ns) {ForwardSessionEventToSession:612(51701ns) ForwardSessionEventToSession:612(3415ns) ForwardSessionEventToSession:612(5088ns) ForwardSessionEventToSession:612(1704ns) ForwardSessionEventToSession:612(3737ns) ForwardSessionEventToSession:612(1986ns) ForwardSessionEventToSession:612(4136ns) ForwardSessionEventToSession:612(4112ns) ForwardSessionEventToSession:612(1569ns) ForwardSessionEventToSession:612(2459ns)}}}
2025-11-11T17:18:13.675757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518359706118661:2090] [node 5] ICP18 handshake (actor [4:7571518359706118689:2113]) is too old
2025-11-11T17:18:13.699691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118702:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.714396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.729904Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518359706118702:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.730060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518355411151320:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.738296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.754807Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518359706118702:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:13.755112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518359706118703:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.763971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518359706118648:2079] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.778044Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5703) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:13.778785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518355411151320:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.778850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:13.778892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518355411151320:2049] [node 2] ICP22 created new session: [4:7571518359706118707:2049]
2025-11-11T17:18:13.778942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518359706118707:2049] [node 2] ICS09 handshake done sender: [4:7571518359706118702:2118] self: [4:7571518359706118704:0] peer: [2:7571518361293205192:0] socket: 112 qp: -1
2025-11-11T17:18:13.778977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518359706118707:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:13.779009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:13.779268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518355411151320:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:13.826140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118701:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.826534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118709:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.854967Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518359706118668:2094] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:13.856671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518359706118709:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.862550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518355411151321:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.862646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:13.862700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518355411151321:2050] [node 3] ICP22 created new session: [4:7571518359706118711:2050]
2025-11-11T17:18:13.862771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518359706118711:2050] [node 3] ICS09 handshake done sender: [4:7571518359706118668:2094] self: [4:7571518359706118695:0] peer: [3:7571518358276670091:0] socket: 74 qp: -1
2025-11-11T17:18:13.862815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518359706118711:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:13.862945Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:13.863091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:13.869453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518355411151319:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.878399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118713:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.914108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518355411151321:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(51329314ns) {HandleHandshakeStatus:329(51325285ns) {DropIncomingHandshake:441(1465ns) DropOutgoingHandshake:456(4987ns) SwitchToState:178(705ns) ProcessPendingSessionEvents:495(19789548ns) {ForwardSessionEventToSession:612(19719147ns) ForwardSessionEventToSession:612(29520ns) ForwardSessionEventToSession:612(2354ns) ForwardSessionEventToSession:612(4569ns) ForwardSessionEventToSession:612(2166ns) ForwardSessionEventToSession:612(4498ns) ForwardSessionEventToSession:612(2569ns) ForwardSessionEventToSession:612(2505ns) ForwardSessionEventToSession:612(4719ns) ForwardSessionEventToSession:612(4465ns)}}}
2025-11-11T17:18:13.914206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518355411151321:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:13.925367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518359706118703:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:13.941983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518359706118648:2079] [node 6] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.942101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:13.942151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518359706118648:2079] [node 6] ICP22 created new session: [4:7571518359706118714:2079]
2025-11-11T17:18:13.942213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518359706118714:2079] [node 6] ICS09 handshake done sender: [4:7571518359706118703:2119] self: [4:7571518359706118706:0] peer: [6:7571518360919126525:0] socket: 116 qp: -1
2025-11-11T17:18:13.942251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518359706118714:2079] [node 6] ICS10 traffic start
2025-11-11T17:18:13.942298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:13.978655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518359706118716:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41751
2025-11-11T17:18:13.790764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163467:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.790924Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163468:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.792081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518360836163462:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47161
2025-11-11T17:18:13.793883Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518360836163454:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:13.799652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518360836163465:2079] [node 6] ICS09 handshake done sender: [1:7571518360836163450:2111] self: [1:7571518360836163449:0] peer: [6:7571518360919126532:0] socket: 79 qp: -1
2025-11-11T17:18:13.804201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518360836163465:2079] [node 6] ICS10 traffic start
2025-11-11T17:18:13.627805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518358276670089:2080] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.812786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518360836163447:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.812869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:13.637267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518362571637438:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4095
2025-11-11T17:18:13.812914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518360836163447:2082] [node 5] ICP22 created new session: [1:7571518360836163473:2082]
2025-11-11T17:18:13.637706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518358276670057:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.812952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518360836163473:2082] [node 5] ICS09 handshake done sender: [1:7571518360836163454:2112] self: [1:7571518360836163453:0] peer: [5:7571518363250296517:0] socket: 94 qp: -1
2025-11-11T17:18:13.812978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518360836163473:2082] [node 5] ICS10 traffic start
2025-11-11T17:18:13.637723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518358276670093:2081] [node 6] ICP35 event processing took too much time PendingConnection:278(16895847ns) {HandleHandshakeStatus:329(672721ns) {DropIncomingHandshake:441(1053ns) DropOutgoingHandshake:456(933ns) SwitchToState:178(904ns) ProcessPendingSessionEvents:495(103082ns) {ForwardSessionEventToSession:612(54283ns) ForwardSessionEventToSession:612(5457ns) ForwardSessionEventToSession:612(4662ns) ForwardSessionEventToSession:612(2250ns) ForwardSessionEventToSession:612(4149ns) ForwardSessionEventToSession:612(1876ns) ForwardSessionEventToSession:612(4364ns) ForwardSessionEventToSession:612(1380ns) ForwardSessionEventToSession:612(1868ns) ForwardSessionEventToSession:612(9472ns)}}}
2025-11-11T17:18:13.647789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518358276670089:2080] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:13.657781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518358276670092:2082] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.657822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518358276670092:2082] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.658566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518358276670057:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(63639308ns) {IncomingHandshake:261(37879138ns) {DropIncomingHandshake:441(598ns) PrepareNewSessionHandshake:476(8446ns) {SwitchToState:178(3876ns)} GenerateSessionVirtualId:988(781ns) IssueIncomingHandshakeReply:193(20027528ns) {SwitchToState:178(2846ns)}}}
2025-11-11T17:18:13.658894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518362571637440:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4095
2025-11-11T17:18:13.673359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518358276670086:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.943552Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:13.813042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:13.943882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518356998237810:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518361293205204:2119] held: no
2025-11-11T17:18:13.961942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518356998237810:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.977916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:13.981824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518356998237810:2048] [node 1] ICP22 created new session: [2:7571518361293205207:2048]
2025-11-11T17:18:13.981887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518361293205207:2048] [node 1] ICS09 handshake done sender: [2:7571518361293205204:2119] self: [2:7571518361293205205:0] peer: [1:7571518360836163457:0] socket: 123 qp: -1
2025-11-11T17:18:13.981938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518361293205207:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:13.981995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:13.982277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518356998237810:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20243975ns) {HandleHandshakeStatus:329(20240105ns) {DropIncomingHandshake:441(879ns) DropOutgoingHandshake:456(801ns) SwitchToState:178(979ns) ProcessPendingSessionEvents:495(1341ns)}}
2025-11-11T17:18:13.673407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:13.988040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:13.814863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:13.998362Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518359706118709:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:14.007436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518355411151319:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:14.007516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:13.678930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518356624159153:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5703) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518360919126522:2113] held: no
2025-11-11T17:18:13.701642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126533:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.708774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518360919126536:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.709097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518360919126472:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.743042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518360919126522:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:13.743414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518356624159153:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.743483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:13.743521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518356624159153:2049] [node 2] ICP22 created new session: [6:7571518360919126542:2049]
2025-11-11T17:18:13.743579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518360919126542:2049] [node 2] ICS09 handshake done sender: [6:7571518360919126522:2113] self: [6:7571518360919126528:0] peer: [2:7571518361293205156:0] socket: 80 qp: -1
2025-11-11T17:18:13.743641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518360919126542:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:13.743682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:13.744170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126541:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.763733Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518360919126536:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:13.765466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518360919126534:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37234
2025-11-11T17:18:13.790045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518360919126472:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.790105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:13.799127Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518360919126524:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:13.799353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518360919126472:2067] [node 5] ICP22 created new session: [6:7571518360919126544:2067]
2025-11-11T17:18:13.811323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518356624159152:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.811370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:13.811414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518356624159152:2048] [node 1] ICP22 created new session: [6:7571518360919126545:2048]
2025-11-11T17:18:13.811457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518360919126545:2048] [node 1] ICS09 handshake done sender: [6:7571518360919126524:2115] self: [6:7571518360919126532:0] peer: [1:7571518360836163449:0] socket: 82 qp: -1
2025-11-11T17:18:13.811506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518360919126545:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:13.811540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:13.821823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518360919126544:2067] [node 5] ICS09 handshake done sender: [6:7571518360919126536:2121] self: [6:7571518360919126539:0] peer: [5:7571518363250296513:0] socket: 110 qp: -1
2025-11-11T17:18:14.007592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518355411151319:2048] [node 1] ICP22 created new session: [4:7571518364001086013:2048]
2025-11-11T17:18:14.007637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518364001086013:2048] [node 1] ICS09 handshake done sender: [4:7571518359706118709:2121] self: [4:7571518359706118710:0] peer: [1:7571518360836163466:0] socket: 76 qp: -1
2025-11-11T17:18:14.007688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518364001086013:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:14.007721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:13.817666Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163461:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.673483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518358276670086:2074] [node 5] ICP22 created new session: [3:7571518362571637442:2074]
2025-11-11T17:18:13.824334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518360919126544:2067] [node 5] ICS10 traffic start
2025-11-11T17:18:13.825939Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518360919126526:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.825977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [6:7571518360919126526:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.830043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:13.830506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518360919126472:2067] [node 5] ICP35 event processing took too much time PendingConnection:278(40262621ns) {HandleHandshakeStatus:329(40258948ns) {DropIncomingHandshake:441(781ns) DropOutgoingHandshake:456(4496ns) SwitchToState:178(565ns) ProcessPendingSessionEvents:495(107910ns) {ForwardSessionEventToSession:612(61170ns) ForwardSessionEventToSession:612(5498ns) ForwardSessionEventToSession:612(2395ns) ForwardSessionEventToSession:612(10089ns) ForwardSessionEventToSession:612(2098ns) ForwardSessionEventToSession:612(1985ns) ForwardSessionEventToSession:612(7375ns) ForwardSessionEventToSession:612(1558ns) ForwardSessionEventToSession:612(2081ns) ForwardSessionEventToSession:612(4741ns)}}}
2025-11-11T17:18:13.886543Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518360919126526:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:13.922235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7571518360919126462:2059] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.930243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:13.930312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518360919126462:2059] [node 4] ICP22 created new session: [6:7571518360919126549:2059]
2025-11-11T17:18:13.930362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518360919126549:2059] [node 4] ICS09 handshake done sender: [6:7571518360919126526:2116] self: [6:7571518360919126525:0] peer: [4:7571518359706118706:0] socket: 115 qp: -1
2025-11-11T17:18:13.930400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518360919126549:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:13.930458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:13.988337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518358955329133:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(44338783ns) {HandleHandshakeStatus:329(44336955ns) {DropIncomingHandshake:441(827ns) DropOutgoingHandshake:456(329ns) SwitchToState:178(528ns) ProcessPendingSessionEvents:495(1138ns)}}
2025-11-11T17:18:13.673506Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518358276670092:2082] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:13.676656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518362571637442:2074] [node 5] ICS09 handshake done sender: [3:7571518358276670089:2080] self: [3:7571518358276670088:0] peer: [5:7571518363250296503:0] socket: 57 qp: -1
2025-11-11T17:18:13.676701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518362571637442:2074] [node 5] ICS10 traffic start
2025-11-11T17:18:13.676743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:13.678473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518358276670087:2079] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.678512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:13.683015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518362571637438:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.693809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518358276670087:2079] [node 4] ICP22 created new session: [3:7571518362571637444:2079]
2025-11-11T17:18:13.695929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518362571637440:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.696273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518358276670086:2074] [node 5] ICP18 handshake (actor [3:7571518362571637440:2118]) is too old
2025-11-11T17:18:13.696366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518362571637446:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4095
2025-11-11T17:18:13.705742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518362571637444:2079] [node 4] ICS09 handshake done sender: [3:7571518358276670092:2082] self: [3:7571518358276670091:0] peer: [4:7571518359706118695:0] socket: 56 qp: -1
2025-11-11T17:18:13.705797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518362571637444:2079] [node 4] ICS10 traffic start
2025-11-11T17:18:13.705869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:13.718289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518358276670093:2081] [node 6] ICP18 handshake (actor [3:7571518362571637428:2114]) is too old
2025-11-11T17:18:13.719106Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518362571637423:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:13.719326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518358276670057:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.719379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:13.719416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518358276670057:2049] [node 2] ICP22 created new session: [3:7571518362571637448:2049]
2025-11-11T17:18:13.719458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518362571637448:2049] [node 2] ICS09 handshake done sender: [3:7571518362571637423:2111] self: [3:7571518362571637437:0] peer: [2:7571518361293205149:0] socket: 69 qp: -1
2025-11-11T17:18:13.719516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518362571637448:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:13.719552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:13.783944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518362571637450:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4095
2025-11-11T17:18:13.825829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518358276670087:2079] [node 4] ICP35 event processing took too much time PendingConnection:278(67389539ns) {HandleHandshakeStatus:329(67386248ns) {DropIncomingHandshake:441(486ns) DropOutgoingHandshake:456(555ns) SwitchToState:178(438ns) ProcessPendingSessionEvents:495(8124067ns) {ForwardSessionEventToSession:612(8041678ns) ForwardSessionEventToSession:612(10642ns) ForwardSessionEventToSession:612(34481ns) ForwardSessionEventToSession:612(7151ns) ForwardSessionEventToSession:612(2049ns) ForwardSessionEventToSession:612(2169ns) ForwardSessionEventToSession:612(3931ns) ForwardSessionEventToSession:612(2072ns) ForwardSessionEventToSession:612(1855ns) ForwardSessionEventToSession:612(3919ns)}}}
2025-11-11T17:18:13.833697Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518362571637450:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.834063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518358276670087:2079] [node 4] ICP18 handshake (actor [3:7571518362571637438:2117]) is too old
2025-11-11T17:18:13.865756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518358276670056:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:13.865895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518358276670056:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28026455ns) {IncomingHandshake:261(28023818ns) {DropIncomingHandshake:441(341ns) PrepareNewSessionHandshake:476(6349ns) {SwitchToState:178(2988ns)} GenerateSessionVirtualId:988(921ns) IssueIncomingHandshakeReply:193(8100039ns) {SwitchToState:178(1345ns)}}}
2025-11-11T17:18:13.899355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518362571637452:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4095
2025-11-11T17:18:13.910699Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518362571637450:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:13.910980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518358276670056:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:13.911033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:13.911078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518358276670056:2048] [node 1] ICP22 created new session: [3:7571518362571637453:2048]
2025-11-11T17:18:13.911115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518362571637453:2048] [node 1] ICS09 handshake done sender: [3:7571518362571637450:2120] self: [3:7571518362571637451:0] peer: [1:7571518360836163460:0] socket: 111 qp: -1
2025-11-11T17:18:13.911148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518362571637453:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:13.911182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:13.926108Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:13.931676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518358276670056:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:13.822087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163462:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.857987Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163459:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.858032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518360836163459:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.866559Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163463:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.866581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518360836163463:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.868333Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518360836163463:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:13.868657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518356541196088:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.868717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:13.868760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518356541196088:2049] [node 3] ICP22 created new session: [1:7571518360836163481:2049]
2025-11-11T17:18:13.868806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518360836163481:2049] [node 3] ICS09 handshake done sender: [1:7571518360836163463:2116] self: [1:7571518360836163460:0] peer: [3:7571518362571637451:0] socket: 106 qp: -1
2025-11-11T17:18:13.868839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518360836163481:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:13.868866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:13.881871Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518360836163459:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:13.898065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518360836163446:2079] [node 6] ICP35 event processing took too much time PendingConnection:278(133976466ns) {HandleHandshakeStatus:329(133973525ns) {DropIncomingHandshake:441(2452ns) DropOutgoingHandshake:456(915ns) SwitchToState:178(500ns) ProcessPendingSessionEvents:495(50071224ns) {ForwardSessionEventToSession:612(18941009ns) ForwardSessionEventToSession:612(8211ns) ForwardSessionEventToSession:612(2669ns) ForwardSessionEventToSession:612(8246981ns) ForwardSessionEventToSession:612(3557ns) ForwardSessionEventToSession:612(7934840ns) ForwardSessionEventToSession:612(8254ns) ForwardSessionEventToSession:612(6666249ns) ForwardSessionEventToSession:612(2662115ns) ForwardSessionEventToSession:612(5564131ns)}}}
2025-11-11T17:18:13.898157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518360836163446:2079] [node 6] ICP18 handshake (actor [1:7571518360836163468:2118]) is too old
2025-11-11T17:18:13.910007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518360836163470:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:13.910042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518360836163470:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:13.914937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518356541196087:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:13.915011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:13.915058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518356541196087:2048] [node 2] ICP22 created new session: [1:7571518360836163483:2048]
2025-11-11T17:18:13.915103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518360836163483:2048] [node 2] ICS09 handshake done sender: [1:7571518360836163459:2113] self: [1:7571518360836163457:0] peer: [2:7571518361293205205:0] socket: 122 qp: -1
2025-11-11T17:18:13.915137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518360836163483:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:13.915168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:13.982991Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518360836163470:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:13.990038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518360836163452:2080] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:14.022218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:14.022822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518360836163452:2080] [node 4] ICP22 created new session: [1:7571518365131130782:2080]
2025-11-11T17:18:14.022882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518365131130782:2080] [node 4] ICS09 handshake done sender: [1:7571518360836163470:2119] self: [1:7571518360836163466:0] peer: [4:7571518359706118710:0] socket: 75 qp: -1
2025-11-11T17:18:14.022917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518365131130782:2080] [node 4] ICS10 traffic start
2025-11-11T17:18:14.022973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:14.023491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518360836163452:2080] [node 4] ICP35 event processing took too much time PendingConnection:278(33200197ns) {HandleHandshakeStatus:329(33197403ns) {DropIncomingHandshake:441(11611017ns) DropOutgoingHandshake:456(2345ns) SwitchToState:178(596ns) ProcessPendingSessionEvents:495(97729ns) {ForwardSessionEventToSession:612(52759ns) ForwardSessionEventToSession:612(3013ns) ForwardSessionEventToSession:612(2118ns) ForwardSessionEventToSession:612(10606ns) ForwardSessionEventToSession:612(2655ns) ForwardSessionEventToSession:612(2043ns) ForwardSessionEventToSession:612(4601ns) ForwardSessionEventToSession:612(1833ns) ForwardSessionEventToSession:612(6806ns) ForwardSessionEventToSession:612(1911ns)}}}
2025-11-11T17:18:14.064787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:14.064989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518361293205207:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-11T17:18:14.065030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518356998237810:2048] [node 1] ICP30 unregister session Session# [2:7571518361293205207:2048] VirtualId# [2:7571518361293205205:0]
2025-11-11T17:18:14.065063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518361293205207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.065132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:14.065168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:14.073759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:14.073979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518362571637453:2048] [node 1] ICS07 socket disconnect 111 reason# EndOfStream
2025-11-11T17:18:14.074011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518362571637453:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.074117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:14.074157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518362571637453:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:14.075103Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Couldn't connect to any resolved address
2025-11-11T17:18:14.075644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518358276670056:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:14.075689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518362571637453:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:14.075719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518358276670056:2048] [node 1] ICP30 unregister session Session# [3:7571518362571637453:2048] VirtualId# [3:7571518362571637451:0]
2025-11-11T17:18:14.075759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:14.075783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518358276670056:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:14.075795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:14.085752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:14.086022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518363250296532:2048] [node 1] ICS01 socket: 96 reason# EndOfStream
2025-11-11T17:18:14.086060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518358955329133:2048] [node 1] ICP30 unregister session Session# [5:7571518363250296532:2048] VirtualId# [5:7571518363250296517:0]
2025-11-11T17:18:14.086093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518363250296532:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.086207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:14.086251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:14.091806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518358276670056:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:14.089799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:14.090027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518364001086013:2048] [node 1] ICS01 socket: 76 reason# EndOfStream
2025-11-11T17:18:14.090068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518355411151319:2048] [node 1] ICP30 unregister session Session# [4:7571518364001086013:2048] VirtualId# [4:7571518359706118710:0]
2025-11-11T17:18:14.090100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518364001086013:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.090204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:14.090252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:14.093781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:14.094105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518360919126545:2048] [node 1] ICS01 socket: 82 reason# EndOfStream
2025-11-11T17:18:14.094182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518356624159152:2048] [node 1] ICP30 unregister session Session# [6:7571518360919126545:2048] VirtualId# [6:7571518360919126532:0]
2025-11-11T17:18:14.094213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518360919126545:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.094315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:14.094403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:14.157796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:14.158011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518362571637448:2049] [node 2] ICS01 socket: 69 reason# EndOfStream
2025-11-11T17:18:14.158059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518358276670057:2049] [node 2] ICP30 unregister session Session# [3:7571518362571637448:2049] VirtualId# [3:7571518362571637437:0]
2025-11-11T17:18:14.158093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518362571637448:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.158194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:14.158240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:14.165775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:14.165797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:14.166010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518360919126542:2049] [node 2] ICS01 socket: 80 reason# EndOfStream
2025-11-11T17:18:14.166018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518363250296518:2049] [node 2] ICS01 socket: 73 reason# EndOfStream
2025-11-11T17:18:14.166051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518356624159153:2049] [node 2] ICP30 unregister session Session# [6:7571518360919126542:2049] VirtualId# [6:7571518360919126528:0]
2025-11-11T17:18:14.166056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518358955329134:2049] [node 2] ICP30 unregister session Session# [5:7571518363250296518:2049] VirtualId# [5:7571518363250296510:0]
2025-11-11T17:18:14.166082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518360919126542:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.166169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:14.166084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518363250296518:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.166208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:14.166194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:14.166243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:14.173711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:14.173968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518359706118707:2049] [node 2] ICS01 socket: 112 reason# EndOfStream
2025-11-11T17:18:14.174017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518355411151320:2049] [node 2] ICP30 unregister session Session# [4:7571518359706118707:2049] VirtualId# [4:7571518359706118704:0]
2025-11-11T17:18:14.174048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518359706118707:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.189723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:14.197769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:14.217806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:14.217804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:14.218044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518359706118711:2050] [node 3] ICS07 socket disconnect 74 reason# EndOfStream
2025-11-11T17:18:14.218076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518359706118711:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.218185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:14.218234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518359706118711:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:14.225959Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65295) Couldn't connect to any resolved address
2025-11-11T17:18:14.226296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518355411151321:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65295) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:14.226351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518359706118711:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:14.226386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518355411151321:2050] [node 3] ICP30 unregister session Session# [4:7571518359706118711:2050] VirtualId# [4:7571518359706118695:0]
2025-11-11T17:18:14.226440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:14.226496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518355411151321:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:14.226511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:14.218094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518360919126535:2050] [node 3] ICS01 socket: 81 reason# EndOfStream
2025-11-11T17:18:14.217742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:14.217981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518363250296526:2050] [node 3] ICS01 socket: 58 reason# EndOfStream
2025-11-11T17:18:14.218028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518358955329135:2050] [node 3] ICP30 unregister session Session# [5:7571518363250296526:2050] VirtualId# [5:7571518363250296503:0]
2025-11-11T17:18:14.218059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518363250296526:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.218137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:14.218179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:14.218140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518356624159154:2050] [node 3] ICP30 unregister session Session# [6:7571518360919126535:2050] VirtualId# [6:7571518360919126527:0]
2025-11-11T17:18:14.218165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518360919126535:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.218249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:14.218299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:14.241732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518355411151321:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:14.313740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:14.314051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518360919126549:2059] [node 4] ICS01 socket: 115 reason# EndOfStream
2025-11-11T17:18:14.314116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518360919126462:2059] [node 4] ICP30 unregister session Session# [6:7571518360919126549:2059] VirtualId# [6:7571518360919126525:0]
2025-11-11T17:18:14.314163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518360919126549:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.314305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:14.314402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:14.394033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:14.394292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518360919126544:2067] [node 5] ICS07 socket disconnect 110 reason# EndOfStream
2025-11-11T17:18:14.394398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518360919126544:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:14.394507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:14.394554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518360919126544:2067] [node 5] ICS15 start handshake
2025-11-11T17:18:14.395403Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2804) Couldn't connect to any resolved address
2025-11-11T17:18:14.395650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518360919126472:2067] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2804) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:14.395695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518360919126544:2067] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:14.395731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518360919126472:2067] [node 5] ICP30 unregister session Session# [6:7571518360919126544:2067] VirtualId# [6:7571518360919126539:0]
2025-11-11T17:18:14.395776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:18:14.395807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518360919126472:2067] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2804) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:14.395818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2804) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:14.405745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518360919126472:2067] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:14.513927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518366731583706:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:14.513977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518366731583707:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:14.529846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518364006098653:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:14.529897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518364006098654:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:14.549934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518365790311133:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:14.549989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518365790311134:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:14.586769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518366710626206:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:14.586812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518366710626207:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:14.586835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518366710626208:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:14.633777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518367421024159:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:14.633828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518367421024160:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:14.633850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518367421024161:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:14.639158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518367144461870:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:14.639191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518367144461871:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:14.639380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518367144461872:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:15.224504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518367144461871:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(104727082ns) {Configure:104(104717326ns) {ProcessConfigured:135(64460ns) {StartInitialHandshake:157(61851ns) {DropHandshakes:469(2383ns) {DropIncomingHandshake:441(522ns) DropOutgoingHandshake:456(387ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:476(1356ns) {SwitchToState:178(456ns)}}}}}
2025-11-11T17:18:15.224974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518371439429184:2060] [node 4] ICP01 ready to work
2025-11-11T17:18:15.230017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518368301066006:2089] [node 4] ICP01 ready to work
2025-11-11T17:18:15.273711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518371005593563:2087] [node 6] ICP01 ready to work
2025-11-11T17:18:15.313966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518371005593567:2088] [node 5] ICP01 ready to work
2025-11-11T17:18:15.341641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518371715991505:2068] [node 6] ICP01 ready to work
2025-11-11T17:18:15.298641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518368301066013:2092] [node 5] ICP01 ready to work
2025-11-11T17:18:15.337890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518371026551065:2104] [node 5] ICP01 ready to work
2025-11-11T17:18:15.385820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518370085278492:2066] [node 4] ICP01 ready to work
2025-11-11T17:18:15.394166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518371005593567:2088] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(76833169ns) {Configure:104(76830207ns) {ProcessConfigured:135(56041ns) {StartInitialHandshake:157(53721ns) {DropHandshakes:469(1886ns) {DropIncomingHandshake:441(355ns) DropOutgoingHandshake:456(262ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:476(945ns) {SwitchToState:178(333ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.410072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518371439429184:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(184707459ns) {Configure:104(184700904ns) {ProcessConfigured:135(53442ns) {StartInitialHandshake:157(51289ns) {DropHandshakes:469(2105ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(405ns)} GenerateSessionVirtualId:988(747ns) PrepareNewSessionHandshake:476(977ns) {SwitchToState:178(468ns)}}}}}
2025-11-11T17:18:15.386157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518370085278493:2067] [node 5] ICP01 ready to work
2025-11-11T17:18:15.397733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518371715991519:2069] [node 4] ICP01 ready to work
2025-11-11T17:18:15.411568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518371439429234:2082] [node 5] ICP01 ready to work
2025-11-11T17:18:15.414626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518368301066014:2093] [node 6] ICP01 ready to work
2025-11-11T17:18:15.386397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518370085278494:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:15.403969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518370085278505:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27498
2025-11-11T17:18:15.449789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518371026551066:2105] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.471463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593581:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.479887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518371005593581:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.480183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518366710626208:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.491171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593583:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
2025-11-11T17:18:15.491936Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518371005593581:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:15.492124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518366710626208:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.492180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:15.492221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518366710626208:2050] [node 3] ICP22 created new session: [4:7571518371005593584:2050]
2025-11-11T17:18:15.492262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518371005593584:2050] [node 3] ICS09 handshake done sender: [4:7571518371005593581:2116] self: [4:7571518371005593582:0] peer: [3:7571518370085278495:0] socket: 80 qp: -1
2025-11-11T17:18:15.492294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518371005593584:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:15.492323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:15.450116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518371026551067:2106] [node 6] ICP01 ready to work
2025-11-11T17:18:15.459154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518367421024160:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(39281291ns) {Configure:104(39274029ns) {ProcessConfigured:135(68440ns) {StartInitialHandshake:157(65194ns) {DropHandshakes:469(2142ns) {DropIncomingHandshake:441(436ns) DropOutgoingHandshake:456(368ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:476(1372ns) {SwitchToState:178(454ns)}}}}}
2025-11-11T17:18:15.467138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518368301066021:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52823
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.478266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518370085278509:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27498
2025-11-11T17:18:15.517920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518371439429234:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(104788937ns) {Configure:104(104786739ns) {ProcessConfigured:135(23831194ns) {StartInitialHandshake:157(23827764ns) {DropHandshakes:469(1659ns) {DropIncomingHandshake:441(317ns) DropOutgoingHandshake:456(509ns)} GenerateSessionVirtualId:988(550ns) PrepareNewSessionHandshake:476(5392ns) {SwitchToState:178(2075ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.538719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593586:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.542122Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066021:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.542517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518368301066029:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52823
2025-11-11T17:18:15.542570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518368301066030:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52823
2025-11-11T17:18:15.489854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278505:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.558065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518371026551066:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(104730945ns) {Configure:104(104729366ns) {ProcessConfigured:135(11715475ns) {StartInitialHandshake:157(11712704ns) {DropHandshakes:469(1219ns) {DropIncomingHandshake:441(294ns) DropOutgoingHandshake:456(246ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:476(4617ns) {SwitchToState:178(2376ns)}}}}}
2025-11-11T17:18:15.560247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518371026551074:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62745
2025-11-11T17:18:15.560505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518371026551079:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62745
2025-11-11T17:18:15.560629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518371026551080:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62745
2025-11-11T17:18:15.569815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551079:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.581913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518368301066032:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52823
2025-11-11T17:18:15.518538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429235:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.521911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518371715991519:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(60655631ns) {Configure:104(60653478ns) {ProcessConfigured:135(69001ns) {StartInitialHandshake:157(67287ns) {DropHandshakes:469(2300ns) {DropIncomingHandshake:441(533ns) DropOutgoingHandshake:456(378ns)} GenerateSessionVirtualId:988(1008ns) PrepareNewSessionHandshake:476(1243ns) {SwitchToState:178(430ns)}}}}}
2025-11-11T17:18:15.593831Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066032:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.594074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518364006098653:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.489978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278496:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.489998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518370085278496:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.491422Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518370085278496:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:15.545746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518370085278492:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.545809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:15.545860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518370085278492:2066] [node 4] ICP22 created new session: [3:7571518370085278512:2066]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.601707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066029:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.524049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991530:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
2025-11-11T17:18:15.569890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518371439429235:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.626387Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066030:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.626486Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066020:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.626505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518368301066020:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.627751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518368301066035:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52823
2025-11-11T17:18:15.595054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429245:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.595153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518371439429184:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.625918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518371439429245:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.626057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518371439429234:2082] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.545912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518370085278512:2066] [node 4] ICS09 handshake done sender: [3:7571518370085278496:2110] self: [3:7571518370085278495:0] peer: [4:7571518371005593582:0] socket: 70 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.641256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429248:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.641608Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518368301066032:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:15.642171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518368301066020:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:15.642454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518364006098653:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.642508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:15.642543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518364006098653:2048] [node 1] ICP22 created new session: [2:7571518368301066038:2048]
2025-11-11T17:18:15.642586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518368301066038:2048] [node 1] ICS09 handshake done sender: [2:7571518368301066032:2118] self: [2:7571518368301066033:0] peer: [1:7571518371026551068:0] socket: 99 qp: -1
2025-11-11T17:18:15.642614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518368301066038:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:15.642641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:15.642983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518368301066013:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.643021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:15.643057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518368301066013:2092] [node 5] ICP22 created new session: [2:7571518368301066040:2092]
2025-11-11T17:18:15.643101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518368301066040:2092] [node 5] ICS09 handshake done sender: [2:7571518368301066020:2112] self: [2:7571518368301066019:0] peer: [5:7571518371715991537:0] socket: 78 qp: -1
2025-11-11T17:18:15.643130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518368301066040:2092] [node 5] ICS10 traffic start
2025-11-11T17:18:15.643163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:15.533793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991531:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
2025-11-11T17:18:15.598510Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551069:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.598545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518371026551069:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.609848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518371026551069:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.651367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429249:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.545952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518370085278512:2066] [node 4] ICS10 traffic start
2025-11-11T17:18:15.603375Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518371715991531:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.603881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991535:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
2025-11-11T17:18:15.604003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
2025-11-11T17:18:15.604141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518367421024160:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.637760Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518371715991523:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.637784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518371715991523:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.638358Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518371715991530:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.633498Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518371005593573:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.673702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518367421024161:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.633543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518371005593573:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.673816Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518371005593586:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.674376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593588:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
2025-11-11T17:18:15.679017Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518371005593573:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:15.545991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:15.621987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518370085278492:2066] [node 4] ICP35 event processing took too much time PendingConnection:278(76059036ns) {HandleHandshakeStatus:329(76054939ns) {DropIncomingHandshake:441(13806ns) DropOutgoingHandshake:456(426ns) SwitchToState:178(782ns) ProcessPendingSessionEvents:495(24069180ns) {ForwardSessionEventToSession:612(24007498ns) ForwardSessionEventToSession:612(14263ns) ForwardSessionEventToSession:612(2927ns) ForwardSessionEventToSession:612(2388ns) ForwardSessionEventToSession:612(5336ns) ForwardSessionEventToSession:612(2062ns) ForwardSessionEventToSession:612(2703ns) ForwardSessionEventToSession:612(7445ns) ForwardSessionEventToSession:612(2039ns) ForwardSessionEventToSession:612(10756ns)}}}
2025-11-11T17:18:15.685878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278509:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.686236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518370085278514:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27498
2025-11-11T17:18:15.686336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518370085278515:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27498
2025-11-11T17:18:15.686428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518365790311134:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.698964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518367421024161:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18639025ns) {IncomingHandshake:261(18637208ns) {DropIncomingHandshake:441(431ns) PrepareNewSessionHandshake:476(1843ns) {SwitchToState:178(571ns)} GenerateSessionVirtualId:988(490ns) IssueIncomingHandshakeReply:193(16043301ns) {SwitchToState:178(4593ns)}}}
2025-11-11T17:18:15.706379Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551080:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.706475Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551074:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.706635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518366731583706:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.706701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:15.706740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518366731583706:2048] [node 2] ICP22 created new session: [1:7571518371026551086:2048]
2025-11-11T17:18:15.706775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518371026551086:2048] [node 2] ICS09 handshake done sender: [1:7571518371026551069:2110] self: [1:7571518371026551068:0] peer: [2:7571518368301066033:0] socket: 97 qp: -1
2025-11-11T17:18:15.706816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518371026551086:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:15.706854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.686598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518371439429249:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.697738Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066018:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.707954Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518371005593588:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.710049Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278515:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.697780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518368301066018:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.708000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518371005593563:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.710276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518370085278518:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27498
2025-11-11T17:18:15.705911Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518368301066018:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:15.709510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518364006098654:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.709544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:15.709580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518364006098654:2049] [node 3] ICP22 created new session: [2:7571518368301066045:2049]
2025-11-11T17:18:15.714659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518368301066045:2049] [node 3] ICS09 handshake done sender: [2:7571518368301066018:2111] self: [2:7571518368301066017:0] peer: [3:7571518370085278517:0] socket: 77 qp: -1
2025-11-11T17:18:15.717912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518368301066045:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:15.739609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:15.745979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991538:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.775287Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:15.708037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:15.755289Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551073:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.755320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518371026551073:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.774774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518371026551088:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62745
2025-11-11T17:18:15.776389Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518364006098654:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(66673721ns) {HandleHandshakeStatus:329(66671631ns) {DropIncomingHandshake:441(401ns) DropOutgoingHandshake:456(539ns) SwitchToState:178(322ns) ProcessPendingSessionEvents:495(14038576ns) {ForwardSessionEventToSession:612(13988951ns) ForwardSessionEventToSession:612(9447ns) ForwardSessionEventToSession:612(4642ns) ForwardSessionEventToSession:612(2757ns) ForwardSessionEventToSession:612(2154ns) ForwardSessionEventToSession:612(4068ns) ForwardSessionEventToSession:612(2285ns) ForwardSessionEventToSession:612(2073ns) ForwardSessionEventToSession:612(4406ns) ForwardSessionEventToSession:612(4105ns)}}}
2025-11-11T17:18:15.776575Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066023:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.776596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518368301066023:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.708079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518371005593563:2087] [node 6] ICP22 created new session: [4:7571518371005593591:2087]
2025-11-11T17:18:15.708123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518371005593591:2087] [node 6] ICS09 handshake done sender: [4:7571518371005593573:2113] self: [4:7571518371005593572:0] peer: [6:7571518371439429246:0] socket: 68 qp: -1
2025-11-11T17:18:15.708152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518371005593591:2087] [node 6] ICS10 traffic start
2025-11-11T17:18:15.708208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:15.714188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518366710626207:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.777523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593595:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
2025-11-11T17:18:15.739220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518365790311133:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.739462Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518370085278509:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:15.754178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518365790311134:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.754235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:15.754276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518365790311134:2049] [node 2] ICP22 created new session: [3:7571518370085278520:2049]
2025-11-11T17:18:15.754324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518370085278520:2049] [node 2] ICS09 handshake done sender: [3:7571518370085278509:2116] self: [3:7571518370085278517:0] peer: [2:7571518368301066017:0] socket: 79 qp: -1
2025-11-11T17:18:15.754360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518370085278520:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:15.754397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:15.777844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518370085278522:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27498
2025-11-11T17:18:15.778580Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.781300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593596:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
2025-11-11T17:18:15.789023Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518371715991523:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:15.789202Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518368301066023:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:15.789789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518371026551073:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:15.790511Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518370085278515:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:15.790859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551088:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.790915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518366731583707:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.790974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:15.791021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518366731583707:2049] [node 3] ICP22 created new session: [1:7571518371026551093:2049]
2025-11-11T17:18:15.791062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518371026551093:2049] [node 3] ICS09 handshake done sender: [1:7571518371026551073:2112] self: [1:7571518371026551072:0] peer: [3:7571518370085278519:0] socket: 100 qp: -1
2025-11-11T17:18:15.791089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518371026551093:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:15.791155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:15.793951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518368301066006:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.794036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:15.794078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518368301066006:2089] [node 4] ICP22 created new session: [2:7571518368301066048:2089]
2025-11-11T17:18:15.794121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518368301066048:2089] [node 4] ICS09 handshake done sender: [2:7571518368301066023:2114] self: [2:7571518368301066022:0] peer: [4:7571518371005593594:0] socket: 76 qp: -1
2025-11-11T17:18:15.794149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518368301066048:2089] [node 4] ICS10 traffic start
2025-11-11T17:18:15.794178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:15.797916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518367144461871:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.798004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518371439429184:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518371439429235:2113] held: no
2025-11-11T17:18:15.807345Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518371715991535:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.807687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518371715991536:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.807790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518367421024160:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518371715991531:2116] held: no
2025-11-11T17:18:15.807916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518371715991505:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.807981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:15.808029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518371715991505:2068] [node 6] ICP22 created new session: [5:7571518371715991541:2068]
2025-11-11T17:18:15.808073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518371715991541:2068] [node 6] ICS09 handshake done sender: [5:7571518371715991523:2111] self: [5:7571518371715991522:0] peer: [6:7571518371439429247:0] socket: 89 qp: -1
2025-11-11T17:18:15.808109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518371715991541:2068] [node 6] ICS10 traffic start
2025-11-11T17:18:15.808141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:15.808551Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518371715991531:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:15.793295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518365790311133:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.793374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:15.819716Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:15.793420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518365790311133:2048] [node 1] ICP22 created new session: [3:7571518370085278523:2048]
2025-11-11T17:18:15.793456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518370085278523:2048] [node 1] ICS09 handshake done sender: [3:7571518370085278515:2118] self: [3:7571518370085278519:0] peer: [1:7571518371026551072:0] socket: 106 qp: -1
2025-11-11T17:18:15.793486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518370085278523:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:15.793519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:15.809310Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278498:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.809330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518370085278498:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.810155Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278514:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.818011Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518370085278498:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.820831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429250:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.821203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518370085278493:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.821273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:15.821311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518370085278493:2067] [node 5] ICP22 created new session: [3:7571518370085278527:2067]
2025-11-11T17:18:15.821358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518370085278527:2067] [node 5] ICS09 handshake done sender: [3:7571518370085278498:2111] self: [3:7571518370085278497:0] peer: [5:7571518371715991540:0] socket: 71 qp: -1
2025-11-11T17:18:15.821386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518370085278527:2067] [node 5] ICS10 traffic start
2025-11-11T17:18:15.821426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:15.822567Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518371005593588:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:15.822905Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518371005593595:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.822978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518366710626207:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518371005593588:2119] held: no
2025-11-11T17:18:15.823055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518366710626207:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.823092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:15.823130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518366710626207:2049] [node 2] ICP22 created new session: [4:7571518371005593597:2049]
2025-11-11T17:18:15.823167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518371005593597:2049] [node 2] ICS09 handshake done sender: [4:7571518371005593588:2119] self: [4:7571518371005593594:0] peer: [2:7571518368301066022:0] socket: 111 qp: -1
2025-11-11T17:18:15.823192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518371005593597:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:15.823223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.824035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429251:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.832711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518371439429251:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.832772Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518371439429245:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:15.833340Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518371439429235:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.835966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278502:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.835995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518370085278502:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.836908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518370085278529:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27498
2025-11-11T17:18:15.833603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518371439429234:2082] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.833663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:15.833698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518371439429234:2082] [node 5] ICP22 created new session: [6:7571518371439429256:2082]
2025-11-11T17:18:15.833734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518371439429256:2082] [node 5] ICS09 handshake done sender: [6:7571518371439429245:2116] self: [6:7571518371439429247:0] peer: [5:7571518371715991522:0] socket: 94 qp: -1
2025-11-11T17:18:15.833764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518371439429256:2082] [node 5] ICS10 traffic start
2025-11-11T17:18:15.833797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:15.834503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518371439429184:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.834527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:15.834562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518371439429184:2060] [node 4] ICP22 created new session: [6:7571518371439429258:2060]
2025-11-11T17:18:15.834596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518371439429258:2060] [node 4] ICS09 handshake done sender: [6:7571518371439429235:2113] self: [6:7571518371439429246:0] peer: [4:7571518371005593572:0] socket: 73 qp: -1
2025-11-11T17:18:15.834617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518371439429258:2060] [node 4] ICS10 traffic start
2025-11-11T17:18:15.834668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:15.835209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429253:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.835333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429254:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.835448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518367144461872:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.837877Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518370085278502:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:15.830080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518367421024160:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.842093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518371439429253:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.831156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518368301066025:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.830161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:15.830206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518367421024160:2049] [node 2] ICP22 created new session: [5:7571518371715991544:2049]
2025-11-11T17:18:15.830254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518371715991544:2049] [node 2] ICS09 handshake done sender: [5:7571518371715991531:2116] self: [5:7571518371715991537:0] peer: [2:7571518368301066019:0] socket: 86 qp: -1
2025-11-11T17:18:15.830291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518371715991544:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:15.830322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:15.830763Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27243) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.831194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518368301066025:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.841927Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518368301066025:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:15.843159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518368301066014:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.843232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:15.843270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518368301066014:2093] [node 6] ICP22 created new session: [2:7571518368301066051:2093]
2025-11-11T17:18:15.843304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518368301066051:2093] [node 6] ICS09 handshake done sender: [2:7571518368301066025:2115] self: [2:7571518368301066024:0] peer: [6:7571518371439429252:0] socket: 93 qp: -1
2025-11-11T17:18:15.843332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518368301066051:2093] [node 6] ICS10 traffic start
2025-11-11T17:18:15.853909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:15.857887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518366710626206:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.862738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518370085278494:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.862791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:15.862837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518370085278494:2069] [node 6] ICP22 created new session: [3:7571518370085278531:2069]
2025-11-11T17:18:15.862909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518370085278531:2069] [node 6] ICS09 handshake done sender: [3:7571518370085278502:2113] self: [3:7571518370085278501:0] peer: [6:7571518371439429260:0] socket: 113 qp: -1
2025-11-11T17:18:15.862952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518370085278531:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:15.863004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:15.854882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518371715991519:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.870152Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:15.854969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518367421024159:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.855236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518367421024161:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27243) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518371715991530:2115] held: no
2025-11-11T17:18:15.855900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991543:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
2025-11-11T17:18:15.874378Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518371715991530:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:15.874775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518367421024161:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.874859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:15.874909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518367421024161:2050] [node 3] ICP22 created new session: [5:7571518371715991550:2050]
2025-11-11T17:18:15.874957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518371715991550:2050] [node 3] ICS09 handshake done sender: [5:7571518371715991530:2115] self: [5:7571518371715991540:0] peer: [3:7571518370085278497:0] socket: 85 qp: -1
2025-11-11T17:18:15.874993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518371715991550:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:15.875031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:15.886916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518371439429249:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.887545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518367144461871:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518371439429249:2118] held: no
2025-11-11T17:18:15.887631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518367144461871:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.887679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:15.887724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518367144461871:2049] [node 2] ICP22 created new session: [6:7571518371439429262:2049]
2025-11-11T17:18:15.887773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518371439429262:2049] [node 2] ICS09 handshake done sender: [6:7571518371439429249:2118] self: [6:7571518371439429252:0] peer: [2:7571518368301066024:0] socket: 110 qp: -1
2025-11-11T17:18:15.887821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518371439429262:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:15.887886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:15.888245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518367144461870:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:15.888351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429261:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.889703Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518371439429251:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:15.889970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518370085278529:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.890081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518370085278494:2069] [node 6] ICP18 handshake (actor [3:7571518370085278529:2121]) is too old
2025-11-11T17:18:15.890800Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27243) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:15.891647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518367144461872:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.891698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:15.891738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518367144461872:2050] [node 3] ICP22 created new session: [6:7571518371439429265:2050]
2025-11-11T17:18:15.891781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518371439429265:2050] [node 3] ICS09 handshake done sender: [6:7571518371439429251:2120] self: [6:7571518371439429260:0] peer: [3:7571518370085278501:0] socket: 115 qp: -1
2025-11-11T17:18:15.891810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518371439429265:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:15.891839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:15.892088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518367144461872:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.905966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551071:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.906005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518371026551071:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.906730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551078:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.906747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518371026551078:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.907812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518371026551097:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62745
2025-11-11T17:18:15.908091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518371005593575:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.908123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518371005593575:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.909894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593600:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.911036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991552:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
2025-11-11T17:18:15.911105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518371715991553:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41293
2025-11-11T17:18:15.911399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518371439429267:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51520
2025-11-11T17:18:15.911769Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518371005593575:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:15.912308Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518371715991535:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:15.912573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518371715991519:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.912659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:15.912700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518371715991519:2069] [node 4] ICP22 created new session: [5:7571518371715991554:2069]
2025-11-11T17:18:15.912765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518371715991554:2069] [node 4] ICS09 handshake done sender: [5:7571518371715991535:2117] self: [5:7571518371715991548:0] peer: [4:7571518371005593574:0] socket: 102 qp: -1
2025-11-11T17:18:15.912804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518371715991554:2069] [node 4] ICS10 traffic start
2025-11-11T17:18:15.912836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:15.914942Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518371026551071:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:15.915237Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518371005593600:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.915287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518371005593567:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.915323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:15.915361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518371005593567:2088] [node 5] ICP22 created new session: [4:7571518371005593602:2088]
2025-11-11T17:18:15.915401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518371005593602:2088] [node 5] ICS09 handshake done sender: [4:7571518371005593575:2114] self: [4:7571518371005593574:0] peer: [5:7571518371715991548:0] socket: 92 qp: -1
2025-11-11T17:18:15.915430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518371005593602:2088] [node 5] ICS10 traffic start
2025-11-11T17:18:15.915503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:15.916141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518371005593567:2088] [node 5] ICP18 handshake (actor [4:7571518371005593600:2122]) is too old
2025-11-11T17:18:15.917018Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518371026551078:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:15.917549Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518371439429253:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:15.917812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518367144461870:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.917870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:15.917906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518367144461870:2048] [node 1] ICP22 created new session: [6:7571518371439429268:2048]
2025-11-11T17:18:15.917940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518371439429268:2048] [node 1] ICS09 handshake done sender: [6:7571518371439429253:2121] self: [6:7571518371439429264:0] peer: [1:7571518371026551077:0] socket: 75 qp: -1
2025-11-11T17:18:15.917972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518371439429268:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:15.918001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:15.919721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551076:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.919752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518371026551076:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:15.920489Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518371026551097:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:15.920534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518371026551065:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.920570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:15.920610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518371026551065:2104] [node 5] ICP22 created new session: [1:7571518371026551100:2104]
2025-11-11T17:18:15.920657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518371026551100:2104] [node 5] ICS09 handshake done sender: [1:7571518371026551071:2111] self: [1:7571518371026551070:0] peer: [5:7571518371715991549:0] socket: 98 qp: -1
2025-11-11T17:18:15.920686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518371026551100:2104] [node 5] ICS10 traffic start
2025-11-11T17:18:15.920715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:15.921121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518371026551067:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.921167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:15.921199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518371026551067:2106] [node 6] ICP22 created new session: [1:7571518371026551102:2106]
2025-11-11T17:18:15.921242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518371026551102:2106] [node 6] ICS09 handshake done sender: [1:7571518371026551078:2115] self: [1:7571518371026551077:0] peer: [6:7571518371439429264:0] socket: 74 qp: -1
2025-11-11T17:18:15.921288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518371026551102:2106] [node 6] ICS10 traffic start
2025-11-11T17:18:15.921316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:15.923922Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518371715991536:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:15.924519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518367421024159:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.924587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:15.924628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518367421024159:2048] [node 1] ICP22 created new session: [5:7571518371715991556:2048]
2025-11-11T17:18:15.924677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518371715991556:2048] [node 1] ICS09 handshake done sender: [5:7571518371715991536:2118] self: [5:7571518371715991549:0] peer: [1:7571518371026551070:0] socket: 103 qp: -1
2025-11-11T17:18:15.924705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518371715991556:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:15.924731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:15.932429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518371026551065:2104] [node 5] ICP18 handshake (actor [1:7571518371026551097:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:15.933506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518371005593604:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39723
2025-11-11T17:18:15.933965Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518371026551076:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:15.934456Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518371005593595:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:15.934682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518366710626206:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:15.934743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:15.934780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518366710626206:2048] [node 1] ICP22 created new session: [4:7571518371005593605:2048]
2025-11-11T17:18:15.934823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518371005593605:2048] [node 1] ICS09 handshake done sender: [4:7571518371005593595:2120] self: [4:7571518371005593599:0] peer: [1:7571518371026551075:0] socket: 120 qp: -1
2025-11-11T17:18:15.934853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518371005593605:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:15.934886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:15.936442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518371026551066:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:15.936519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:15.936566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518371026551066:2105] [node 4] ICP22 created new session: [1:7571518371026551105:2105]
2025-11-11T17:18:15.936603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518371026551105:2105] [node 4] ICS09 handshake done sender: [1:7571518371026551076:2114] self: [1:7571518371026551075:0] peer: [4:7571518371005593599:0] socket: 118 qp: -1
2025-11-11T17:18:15.936632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518371026551105:2105] [node 4] ICS10 traffic start
2025-11-11T17:18:15.936669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:15.950291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:15.950503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518368301066038:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-11-11T17:18:15.950542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518364006098653:2048] [node 1] ICP30 unregister session Session# [2:7571518368301066038:2048] VirtualId# [2:7571518368301066033:0]
2025-11-11T17:18:15.950572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518368301066038:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:15.950653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:15.950700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:15.952331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:15.952535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518370085278523:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-11-11T17:18:15.952580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518365790311133:2048] [node 1] ICP30 unregister session Session# [3:7571518370085278523:2048] VirtualId# [3:7571518370085278519:0]
2025-11-11T17:18:15.952609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518370085278523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:15.952680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:15.952716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:15.971603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:15.971820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518371715991556:2048] [node 1] ICS01 socket: 103 reason# EndOfStream
2025-11-11T17:18:15.971872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518367421024159:2048] [node 1] ICP30 unregister session Session# [5:7571518371715991556:2048] VirtualId# [5:7571518371715991549:0]
2025-11-11T17:18:15.971905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518371715991556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:15.971980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:15.972024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:15.974044Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:15.974218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518371439429268:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-11-11T17:18:15.974250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518367144461870:2048] [node 1] ICP30 unregister session Session# [6:7571518371439429268:2048] VirtualId# [6:7571518371439429264:0]
2025-11-11T17:18:15.974272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518371439429268:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:15.973726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:15.981677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:15.981799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:15.973985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518371005593605:2048] [node 1] ICS01 socket: 120 reason# EndOfStream
2025-11-11T17:18:15.974021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518366710626206:2048] [node 1] ICP30 unregister session Session# [4:7571518371005593605:2048] VirtualId# [4:7571518371005593599:0]
2025-11-11T17:18:15.974048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518371005593605:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:15.974123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:15.974159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:16.059029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:16.059253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518370085278520:2049] [node 2] ICS01 socket: 79 reason# EndOfStream
2025-11-11T17:18:16.059290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518365790311134:2049] [node 2] ICP30 unregister session Session# [3:7571518370085278520:2049] VirtualId# [3:7571518370085278517:0]
2025-11-11T17:18:16.059323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518370085278520:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.059398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:16.059438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:16.073753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:16.073982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518371005593597:2049] [node 2] ICS01 socket: 111 reason# EndOfStream
2025-11-11T17:18:16.074028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518366710626207:2049] [node 2] ICP30 unregister session Session# [4:7571518371005593597:2049] VirtualId# [4:7571518371005593594:0]
2025-11-11T17:18:16.074061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518371005593597:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.074148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:16.074205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:16.077710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:16.077898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518371715991544:2049] [node 2] ICS01 socket: 86 reason# EndOfStream
2025-11-11T17:18:16.077937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518367421024160:2049] [node 2] ICP30 unregister session Session# [5:7571518371715991544:2049] VirtualId# [5:7571518371715991537:0]
2025-11-11T17:18:16.077965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518371715991544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.078033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:16.078065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:16.081797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:16.082110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518371439429262:2049] [node 2] ICS01 socket: 110 reason# EndOfStream
2025-11-11T17:18:16.082332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518367144461871:2049] [node 2] ICP30 unregister session Session# [6:7571518371439429262:2049] VirtualId# [6:7571518371439429252:0]
2025-11-11T17:18:16.082438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518371439429262:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.082558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:16.082637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:16.134792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:16.134998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518371715991550:2050] [node 3] ICS01 socket: 85 reason# EndOfStream
2025-11-11T17:18:16.135039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518367421024161:2050] [node 3] ICP30 unregister session Session# [5:7571518371715991550:2050] VirtualId# [5:7571518371715991540:0]
2025-11-11T17:18:16.135070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518371715991550:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.135139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:16.135178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:16.136542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:16.136738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518371439429265:2050] [node 3] ICS01 socket: 115 reason# EndOfStream
2025-11-11T17:18:16.136769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518367144461872:2050] [node 3] ICP30 unregister session Session# [6:7571518371439429265:2050] VirtualId# [6:7571518371439429260:0]
2025-11-11T17:18:16.136794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518371439429265:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.136862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:16.136927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:16.137015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:16.137189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518371005593584:2050] [node 3] ICS01 socket: 80 reason# EndOfStream
2025-11-11T17:18:16.137216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518366710626208:2050] [node 3] ICP30 unregister session Session# [4:7571518371005593584:2050] VirtualId# [4:7571518371005593582:0]
2025-11-11T17:18:16.137237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518371005593584:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.137307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:16.137343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:16.185759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:16.185974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518371715991554:2069] [node 4] ICS01 socket: 102 reason# EndOfStream
2025-11-11T17:18:16.186012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518371715991519:2069] [node 4] ICP30 unregister session Session# [5:7571518371715991554:2069] VirtualId# [5:7571518371715991548:0]
2025-11-11T17:18:16.186041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518371715991554:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.186109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:16.186143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:16.193698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:16.193933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518371439429258:2060] [node 4] ICS01 socket: 73 reason# EndOfStream
2025-11-11T17:18:16.193985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518371439429184:2060] [node 4] ICP30 unregister session Session# [6:7571518371439429258:2060] VirtualId# [6:7571518371439429246:0]
2025-11-11T17:18:16.194018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518371439429258:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:16.194113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:16.194166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:16.354072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518372077053202:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:16.354116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518372077053203:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:16.426606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518372217944279:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:16.426643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518372217944280:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:16.485922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518375306696597:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:16.485960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518375306696598:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:16.530478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518375993859400:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:16.530517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518375993859401:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:16.530540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518375993859402:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:16.574640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518373604900909:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:16.574685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518373604900910:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:16.574707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518373604900911:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:16.778489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518373604900922:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:16.780808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518375993859412:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
2025-11-11T17:18:16.961694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518375306696631:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:16.933445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518373604900939:2063] [node 6] ICP01 ready to work
2025-11-11T17:18:16.952550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518375993859400:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(167709258ns) {Configure:104(167705975ns) {ProcessConfigured:135(34615051ns) {StartInitialHandshake:157(34612093ns) {DropHandshakes:469(2137ns) {DropIncomingHandshake:441(413ns) DropOutgoingHandshake:456(328ns)} GenerateSessionVirtualId:988(1079ns) PrepareNewSessionHandshake:476(4724ns) {SwitchToState:178(2530ns)}}}}}
2025-11-11T17:18:16.989930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518372217944320:2064] [node 5] ICP01 ready to work
2025-11-11T17:18:16.961802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518375306696638:2066] [node 5] ICP01 ready to work
2025-11-11T17:18:17.030604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518373604900939:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(56565262ns) {Configure:104(56559826ns) {ProcessConfigured:135(96477ns) {StartInitialHandshake:157(94626ns) {DropHandshakes:469(2321ns) {DropIncomingHandshake:441(573ns) DropOutgoingHandshake:456(395ns)} GenerateSessionVirtualId:988(1078ns) PrepareNewSessionHandshake:476(1697ns) {SwitchToState:178(761ns)}}}}}
2025-11-11T17:18:16.952713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518375993859412:2059] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:16.998366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518372077053258:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:17.019512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518372217944325:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.001962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518372077053255:2063] [node 4] ICP01 ready to work
2025-11-11T17:18:17.057773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518380288826730:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.066193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518376512911633:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:17.021913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518379601663944:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:16.925976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518373351879253:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:17.053847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518376372020559:2065] [node 5] ICP01 ready to work
2025-11-11T17:18:17.189734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518379601663948:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38977
2025-11-11T17:18:16.926055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518373351879254:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:17.233918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868242:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.183780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518376372020567:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50511
2025-11-11T17:18:17.250775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518380288826733:2077] [node 6] ICP01 ready to work
2025-11-11T17:18:17.075493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518376512911630:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-11T17:18:16.926101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518373351879255:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:17.202513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518379601663949:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38977
2025-11-11T17:18:17.183949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518376372020568:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50511
2025-11-11T17:18:16.926679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518373351879277:2065] [node 4] ICP01 ready to work
2025-11-11T17:18:17.185152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518376512911644:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.203014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518379601663948:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.189893Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020567:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.203162Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911630:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.249815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518379601663949:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.394652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518380288826730:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(65107211ns) {Configure:104(65104276ns) {ProcessConfigured:135(150806ns) {StartInitialHandshake:157(99799ns) {DropHandshakes:469(2025ns) {DropIncomingHandshake:441(388ns) DropOutgoingHandshake:456(434ns)} GenerateSessionVirtualId:988(6020ns) PrepareNewSessionHandshake:476(1548ns) {SwitchToState:178(671ns)}} IncomingHandshake:261(23816ns) {DropIncomingHandshake:441(163ns) PrepareNewSessionHandshake:476(602ns) {SwitchToState:178(444ns)} GenerateSessionVirtualId:988(207ns) IssueIncomingHandshakeReply:193(760ns)}}}}
2025-11-11T17:18:17.396434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518380288826734:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
2025-11-11T17:18:17.396571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518380288826746:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
2025-11-11T17:18:17.396676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518380288826766:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:16.926922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518373351879278:2067] [node 5] ICP01 ready to work
2025-11-11T17:18:17.189961Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020568:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.422184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518375993859402:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(531907670ns) {Configure:104(531903215ns) {ProcessConfigured:135(56835ns) {StartInitialHandshake:157(54420ns) {DropHandshakes:469(2211ns) {DropIncomingHandshake:441(379ns) DropOutgoingHandshake:456(441ns)} GenerateSessionVirtualId:988(1100ns) PrepareNewSessionHandshake:476(1681ns) {SwitchToState:178(853ns)}}}}}
2025-11-11T17:18:17.402367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868284:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.402449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868285:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.274287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518372217944280:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(50428778ns) {IncomingHandshake:261(50426680ns) {DropIncomingHandshake:441(1437ns) PrepareNewSessionHandshake:476(16292383ns) {SwitchToState:178(6767ns)} GenerateSessionVirtualId:988(1204ns) IssueIncomingHandshakeReply:193(10464ns)}}
2025-11-11T17:18:17.358148Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020566:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.358193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518376372020566:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.359076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518376372020578:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50511
2025-11-11T17:18:17.360787Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518376372020566:2109] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:17.361066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518372077053202:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.361107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:17.254694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518379601663944:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(109264603ns) {Configure:104(109261231ns) {ProcessConfigured:135(60840ns) {StartInitialHandshake:157(59096ns) {DropHandshakes:469(2085ns) {DropIncomingHandshake:441(469ns) DropOutgoingHandshake:456(336ns)} GenerateSessionVirtualId:988(732ns) PrepareNewSessionHandshake:476(1581ns) {SwitchToState:178(487ns)}}}}}
2025-11-11T17:18:17.460869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518373351879255:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(478634295ns) {Configure:104(478631450ns) {ProcessConfigured:135(29461944ns) {StartInitialHandshake:157(29459597ns) {DropHandshakes:469(4797ns) {DropIncomingHandshake:441(444ns) DropOutgoingHandshake:456(597ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:476(5729ns) {SwitchToState:178(3194ns)}}}}}
2025-11-11T17:18:17.429890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518380288826734:2080] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.470493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518373351879277:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(292416007ns) {Configure:104(292395023ns) {ProcessConfigured:135(75170ns) {StartInitialHandshake:157(73207ns) {DropHandshakes:469(2306ns) {DropIncomingHandshake:441(439ns) DropOutgoingHandshake:456(482ns)} GenerateSessionVirtualId:988(772ns) PrepareNewSessionHandshake:476(2656ns) {SwitchToState:178(1319ns)}}}}}
2025-11-11T17:18:17.471369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846622:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.471482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846623:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.361140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518372077053202:2048] [node 2] ICP22 created new session: [1:7571518376372020580:2048]
2025-11-11T17:18:17.256807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518375306696598:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.305677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911637:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.361175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518376372020580:2048] [node 2] ICS09 handshake done sender: [1:7571518376372020566:2109] self: [1:7571518376372020565:0] peer: [2:7571518376512911658:0] socket: 87 qp: -1
2025-11-11T17:18:17.361199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518376372020580:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:17.361229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:17.460744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518375993859401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.305726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518376512911637:2103] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.329686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911644:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.355037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518376512911633:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(285394406ns) {Configure:104(285390815ns) {ProcessConfigured:135(72357ns) {StartInitialHandshake:157(70505ns) {DropHandshakes:469(2345ns) {DropIncomingHandshake:441(786ns) DropOutgoingHandshake:456(369ns)} GenerateSessionVirtualId:988(757ns) PrepareNewSessionHandshake:476(1684ns) {SwitchToState:178(566ns)}}}}}
2025-11-11T17:18:17.356244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518372217944279:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.359461Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518376512911637:2103] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:17.360049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518372217944280:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.360105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:17.360143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518372217944280:2049] [node 3] ICP22 created new session: [2:7571518376512911660:2049]
2025-11-11T17:18:17.360182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518376512911660:2049] [node 3] ICS09 handshake done sender: [2:7571518376512911637:2103] self: [2:7571518376512911636:0] peer: [3:7571518379601663961:0] socket: 81 qp: -1
2025-11-11T17:18:17.360210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518376512911660:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:17.360240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:17.366129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518376512911659:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-11T17:18:17.386245Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518376512911644:2106] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:17.422822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518372217944279:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.422873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:17.422921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518372217944279:2048] [node 1] ICP22 created new session: [2:7571518376512911664:2048]
2025-11-11T17:18:17.422967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518376512911664:2048] [node 1] ICS09 handshake done sender: [2:7571518376512911644:2106] self: [2:7571518376512911658:0] peer: [1:7571518376372020565:0] socket: 88 qp: -1
2025-11-11T17:18:17.422999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518376512911664:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:17.423031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:17.441998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518376512911663:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-11T17:18:17.473752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911663:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.494776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020578:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.496512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518376372020583:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50511
2025-11-11T17:18:17.497070Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911639:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.497103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518376512911639:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.501859Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518376512911639:2104] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:17.484315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518375993859401:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(30758153ns) {IncomingHandshake:261(30756131ns) {DropIncomingHandshake:441(351ns) PrepareNewSessionHandshake:476(1627ns) {SwitchToState:178(611ns)} GenerateSessionVirtualId:988(679ns) IssueIncomingHandshakeReply:193(16631159ns) {SwitchToState:178(915ns)}}}
2025-11-11T17:18:17.493671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518380288826746:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.361747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518379601663976:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38977
2025-11-11T17:18:17.449863Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.477872Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518377899868284:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.513666Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518380288826766:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.523339Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518380288826771:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.523360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518380288826771:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.524030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518375993859400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.524274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518375993859402:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.513674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518375306696598:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518379601663949:2098] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.542149Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.543372Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020564:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.543407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518376372020564:2108] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.502600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518372217944325:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.481834Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518377899868285:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.513871Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518377646846622:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.502665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:17.484265Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518377899868242:2074] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.553795Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518376372020564:2108] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:17.559200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518373351879253:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.605395Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518379601663949:2098] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.606425Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518379601663953:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.606455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518379601663953:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.607097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518379601663951:2099] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.502718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518372217944325:2065] [node 4] ICP22 created new session: [2:7571518376512911670:2065]
2025-11-11T17:18:17.559441Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518377646846623:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.607113Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518379601663951:2099] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.570074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518380288826780:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
2025-11-11T17:18:17.489909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518373604900910:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.510880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518373604900922:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.511012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518373604900922:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(20899824ns) {IncomingHandshake:261(20897606ns) {DropIncomingHandshake:441(1199ns) PrepareNewSessionHandshake:476(4154ns) {SwitchToState:178(3188ns)} GenerateSessionVirtualId:988(830ns) IssueIncomingHandshakeReply:193(53499ns) {SwitchToState:178(553ns)}}}
2025-11-11T17:18:17.511095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518373604900911:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.538144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868289:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.583454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868290:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.569751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518376372020559:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(436520231ns) {Configure:104(436517883ns) {ProcessConfigured:135(16363682ns) {StartInitialHandshake:157(16361417ns) {DropHandshakes:469(1037ns) {DropIncomingHandshake:441(222ns) DropOutgoingHandshake:456(259ns)} GenerateSessionVirtualId:988(883ns) PrepareNewSessionHandshake:476(3367ns) {SwitchToState:178(1703ns)}}}}}
2025-11-11T17:18:17.502764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518376512911670:2065] [node 4] ICS09 handshake done sender: [2:7571518376512911639:2104] self: [2:7571518376512911638:0] peer: [4:7571518380288826779:0] socket: 82 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.608302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518379601663977:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38977
2025-11-11T17:18:17.582185Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020583:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.582273Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020562:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.582294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518376372020562:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.605143Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518376372020562:2107] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:17.614121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518372077053255:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.614178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:17.614229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518372077053255:2063] [node 4] ICP22 created new session: [1:7571518376372020591:2063]
2025-11-11T17:18:17.614268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518376372020591:2063] [node 4] ICS09 handshake done sender: [1:7571518376372020564:2108] self: [1:7571518376372020563:0] peer: [4:7571518380288826781:0] socket: 86 qp: -1
2025-11-11T17:18:17.614298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518376372020591:2063] [node 4] ICS10 traffic start
2025-11-11T17:18:17.614326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:17.614678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518372077053258:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.614712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:17.614736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518372077053258:2064] [node 6] ICP22 created new session: [1:7571518376372020593:2064]
2025-11-11T17:18:17.614781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518376372020593:2064] [node 6] ICS09 handshake done sender: [1:7571518376372020562:2107] self: [1:7571518376372020561:0] peer: [6:7571518377646846629:0] socket: 85 qp: -1
2025-11-11T17:18:17.614803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518376372020593:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:17.614823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:17.631151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518376372020595:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50511
2025-11-11T17:18:17.559747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518373351879254:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.626491Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518380288826771:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:17.502797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518376512911670:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:17.502830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:17.510199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518376512911667:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-11T17:18:17.522059Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911640:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.522085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518376512911640:2105] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.524518Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518376512911640:2105] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:17.524767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518372217944320:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.524803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:17.524883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518372217944320:2064] [node 5] ICP22 created new session: [2:7571518376512911673:2064]
2025-11-11T17:18:17.524953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518376512911673:2064] [node 5] ICS09 handshake done sender: [2:7571518376512911640:2105] self: [2:7571518376512911631:0] peer: [5:7571518377899868286:0] socket: 77 qp: -1
2025-11-11T17:18:17.524987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518376512911673:2064] [node 5] ICS10 traffic start
2025-11-11T17:18:17.525018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:17.581823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911667:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.585632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518376512911656:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.585689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518376512911656:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.630135Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518376512911656:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:17.631335Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518377899868284:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:17.584680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518373351879278:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(83632023ns) {Configure:104(83627557ns) {ProcessConfigured:135(33196803ns) {StartInitialHandshake:157(33195179ns) {DropHandshakes:469(1360ns) {DropIncomingHandshake:441(206ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(302ns) PrepareNewSessionHandshake:476(3169ns) {SwitchToState:178(1168ns)}}}}}
2025-11-11T17:18:17.610156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846631:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.642369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846632:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.608407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518379601663978:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38977
2025-11-11T17:18:17.629096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518375993859401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518380288826734:2080] held: no
2025-11-11T17:18:17.685637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020570:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.685699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518376372020570:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.686069Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.613720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518375306696598:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(36516392ns) {HandleHandshakeStatus:395(36513697ns) {LogHandshakeFail:472(17725ns) {UpdateErrorStateLog:544(8619ns)} DropOutgoingHandshake:456(1686ns) ProcessPendingSessionEvents:495(16503504ns) {DropSessionEvent:512(10534ns) DropSessionEvent:512(3406ns) DropSessionEvent:512(4500742ns) DropSessionEvent:512(13758ns) DropSessionEvent:512(3274ns) DropSessionEvent:512(3864ns) DropSessionEvent:512(3216ns) DropSessionEvent:512(3927ns) DropSessionEvent:512(2784ns) DropSessionEvent:512(3951ns)}}}
2025-11-11T17:18:17.615721Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518379601663953:2100] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:17.622521Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518379601663951:2099] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:17.622915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518379601663977:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.622954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518375306696598:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.623015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:17.623140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518375306696598:2049] [node 2] ICP22 created new session: [3:7571518379601663982:2049]
2025-11-11T17:18:17.623221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518379601663982:2049] [node 2] ICS09 handshake done sender: [3:7571518379601663949:2098] self: [3:7571518379601663961:0] peer: [2:7571518376512911636:0] socket: 83 qp: -1
2025-11-11T17:18:17.623263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518379601663982:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:17.623298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:17.623727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518379601663978:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.623763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518375306696638:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.623807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:17.623838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518375306696638:2066] [node 5] ICP22 created new session: [3:7571518379601663984:2066]
2025-11-11T17:18:17.623894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518379601663984:2066] [node 5] ICS09 handshake done sender: [3:7571518379601663953:2100] self: [3:7571518379601663952:0] peer: [5:7571518377899868288:0] socket: 90 qp: -1
2025-11-11T17:18:17.623918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518379601663984:2066] [node 5] ICS10 traffic start
2025-11-11T17:18:17.623950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:17.624278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518375306696631:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.624303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:17.624331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518375306696631:2064] [node 4] ICP22 created new session: [3:7571518379601663986:2064]
2025-11-11T17:18:17.624383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518379601663986:2064] [node 4] ICS09 handshake done sender: [3:7571518379601663951:2099] self: [3:7571518379601663950:0] peer: [4:7571518380288826783:0] socket: 89 qp: -1
2025-11-11T17:18:17.624403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518379601663986:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:17.624425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:17.649826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518375306696597:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.699978Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518377646846622:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:17.658611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518376512911633:2066] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.663126Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16792) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.650880Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.663770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.722823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518379601663991:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38977
2025-11-11T17:18:17.689808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020595:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.677345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518376512911633:2066] [node 6] ICP22 created new session: [2:7571518376512911677:2066]
2025-11-11T17:18:17.669987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518380288826734:2080] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:17.737747Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518376372020570:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:17.741698Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518376372020586:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.741729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518376372020586:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.698091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518373604900910:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.698142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:17.698197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518373604900910:2049] [node 2] ICP22 created new session: [5:7571518377899868293:2049]
2025-11-11T17:18:17.698241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518377899868293:2049] [node 2] ICS09 handshake done sender: [5:7571518377899868284:2116] self: [5:7571518377899868286:0] peer: [2:7571518376512911631:0] socket: 101 qp: -1
2025-11-11T17:18:17.698272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518377899868293:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:17.698301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:17.698969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518373604900911:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16792) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518377899868242:2074] held: no
2025-11-11T17:18:17.699445Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518377899868285:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:17.709960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868291:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.711431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868292:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.711576Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.712408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518373604900922:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518377899868285:2117] held: no
2025-11-11T17:18:17.712475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518373604900922:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.712502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:17.712532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518373604900922:2059] [node 4] ICP22 created new session: [5:7571518377899868296:2059]
2025-11-11T17:18:17.712572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518377899868296:2059] [node 4] ICS09 handshake done sender: [5:7571518377899868285:2117] self: [5:7571518377899868287:0] peer: [4:7571518380288826770:0] socket: 104 qp: -1
2025-11-11T17:18:17.712596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518377899868296:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:17.712626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:17.712902Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518377899868242:2074] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:17.723387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518373604900911:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.723431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:17.723471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518373604900911:2050] [node 3] ICP22 created new session: [5:7571518377899868298:2050]
2025-11-11T17:18:17.723508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518377899868298:2050] [node 3] ICS09 handshake done sender: [5:7571518377899868242:2074] self: [5:7571518377899868288:0] peer: [3:7571518379601663952:0] socket: 91 qp: -1
2025-11-11T17:18:17.723534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518377899868298:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:17.723579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:17.723980Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518377899868292:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.725299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518373604900909:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.701708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518376512911677:2066] [node 6] ICS09 handshake done sender: [2:7571518376512911656:2116] self: [2:7571518376512911655:0] peer: [6:7571518377646846630:0] socket: 80 qp: -1
2025-11-11T17:18:17.713404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518376512911677:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:17.713443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:17.718193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518376512911633:2066] [node 6] ICP35 event processing took too much time PendingConnection:278(59402049ns) {HandleHandshakeStatus:329(59398581ns) {DropIncomingHandshake:441(14653ns) DropOutgoingHandshake:456(4390ns) SwitchToState:178(1166ns) ProcessPendingSessionEvents:495(99662ns) {ForwardSessionEventToSession:612(57233ns) ForwardSessionEventToSession:612(4303ns) ForwardSessionEventToSession:612(5029ns) ForwardSessionEventToSession:612(7815ns) ForwardSessionEventToSession:612(1995ns) ForwardSessionEventToSession:612(4968ns) ForwardSessionEventToSession:612(2085ns) ForwardSessionEventToSession:612(5018ns) ForwardSessionEventToSession:612(1904ns) ForwardSessionEventToSession:612(1803ns)}}}
2025-11-11T17:18:17.743761Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518376372020586:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.758065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518376372020559:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.758161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:17.758206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518376372020559:2065] [node 5] ICP22 created new session: [1:7571518376372020600:2065]
2025-11-11T17:18:17.758249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518376372020600:2065] [node 5] ICS09 handshake done sender: [1:7571518376372020586:2118] self: [1:7571518376372020585:0] peer: [5:7571518377899868300:0] socket: 96 qp: -1
2025-11-11T17:18:17.758277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518376372020600:2065] [node 5] ICS10 traffic start
2025-11-11T17:18:17.758312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:17.758674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518372077053203:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.758713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:17.758751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518372077053203:2049] [node 3] ICP22 created new session: [1:7571518376372020602:2049]
2025-11-11T17:18:17.758782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518376372020602:2049] [node 3] ICS09 handshake done sender: [1:7571518376372020570:2112] self: [1:7571518376372020569:0] peer: [3:7571518379601663989:0] socket: 110 qp: -1
2025-11-11T17:18:17.758805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518376372020602:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:17.758853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.753380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868301:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.768725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518379601663992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38977
2025-11-11T17:18:17.768852Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518377646846623:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:17.686893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518375993859400:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518380288826746:2114] held: no
2025-11-11T17:18:17.687001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518380288826730:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.687052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:17.687100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518380288826730:2069] [node 5] ICP22 created new session: [4:7571518380288826787:2069]
2025-11-11T17:18:17.687138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518380288826787:2069] [node 5] ICS09 handshake done sender: [4:7571518380288826771:2116] self: [4:7571518380288826770:0] peer: [5:7571518377899868287:0] socket: 103 qp: -1
2025-11-11T17:18:17.687171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518380288826787:2069] [node 5] ICS10 traffic start
2025-11-11T17:18:17.687203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:17.687609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518380288826784:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
2025-11-11T17:18:17.705729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518380288826785:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
2025-11-11T17:18:17.710297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518375993859401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.710337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:17.710375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518375993859401:2049] [node 2] ICP22 created new session: [4:7571518380288826789:2049]
2025-11-11T17:18:17.710423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518380288826789:2049] [node 2] ICS09 handshake done sender: [4:7571518380288826734:2080] self: [4:7571518380288826779:0] peer: [2:7571518376512911638:0] socket: 92 qp: -1
2025-11-11T17:18:17.710479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518380288826789:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:17.710509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:17.710908Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518380288826746:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:17.725741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518375993859400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.725802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:17.725831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518375993859400:2048] [node 1] ICP22 created new session: [4:7571518380288826792:2048]
2025-11-11T17:18:17.725882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518380288826792:2048] [node 1] ICS09 handshake done sender: [4:7571518380288826746:2114] self: [4:7571518380288826781:0] peer: [1:7571518376372020563:0] socket: 93 qp: -1
2025-11-11T17:18:17.725908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518380288826792:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:17.725949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:17.734152Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518380288826766:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:17.765539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518375993859400:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(39696900ns) {HandleHandshakeStatus:329(39694867ns) {DropIncomingHandshake:441(263ns) DropOutgoingHandshake:456(275ns) SwitchToState:178(88ns) ProcessPendingSessionEvents:495(1208ns)}}
2025-11-11T17:18:17.767146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518375993859402:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.767198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:17.767249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518375993859402:2050] [node 3] ICP22 created new session: [4:7571518380288826794:2050]
2025-11-11T17:18:17.767285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518380288826794:2050] [node 3] ICS09 handshake done sender: [4:7571518380288826766:2115] self: [4:7571518380288826783:0] peer: [3:7571518379601663950:0] socket: 94 qp: -1
2025-11-11T17:18:17.767315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518380288826794:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:17.767341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:17.769726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518373351879253:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518377646846622:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.774186Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.793891Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.789398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518380288826796:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38499
2025-11-11T17:18:17.795475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518373604900909:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518377899868292:2121] held: no
2025-11-11T17:18:17.795743Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518377899868292:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:17.798126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518373604900909:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.798180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:17.798223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518373604900909:2048] [node 1] ICP22 created new session: [5:7571518377899868302:2048]
2025-11-11T17:18:17.798263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518377899868302:2048] [node 1] ICS09 handshake done sender: [5:7571518377899868292:2121] self: [5:7571518377899868300:0] peer: [1:7571518376372020585:0] socket: 129 qp: -1
2025-11-11T17:18:17.798294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518377899868302:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:17.798325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:17.769801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518373351879253:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.769839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:17.769878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518373351879253:2048] [node 1] ICP22 created new session: [6:7571518377646846634:2048]
2025-11-11T17:18:17.769916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518377646846634:2048] [node 1] ICS09 handshake done sender: [6:7571518377646846622:2114] self: [6:7571518377646846629:0] peer: [1:7571518376372020561:0] socket: 84 qp: -1
2025-11-11T17:18:17.769942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518377646846634:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:17.769970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:17.778256Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.781960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518373351879254:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.782002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:17.782040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518373351879254:2049] [node 2] ICP22 created new session: [6:7571518377646846638:2049]
2025-11-11T17:18:17.782080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518377646846638:2049] [node 2] ICS09 handshake done sender: [6:7571518377646846623:2115] self: [6:7571518377646846630:0] peer: [2:7571518376512911655:0] socket: 106 qp: -1
2025-11-11T17:18:17.782108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518377646846638:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:17.782167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:17.782344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518373351879254:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:17.783219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846633:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.783524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.783603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846637:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.797747Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518377646846633:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.775072Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:17.785581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518375306696597:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518379601663978:2119] held: no
2025-11-11T17:18:17.785881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518379601663978:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:17.801752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518375306696597:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.801802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:17.807043Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518377646846637:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.807257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518373351879255:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.809116Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518377899868241:2073] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.809146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518377899868241:2073] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.809385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518377899868304:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29476
2025-11-11T17:18:17.809557Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518380288826796:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.808741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518373351879278:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.806812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518375306696597:2048] [node 1] ICP22 created new session: [3:7571518379601663993:2048]
2025-11-11T17:18:17.814727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518379601663993:2048] [node 1] ICS09 handshake done sender: [3:7571518379601663978:2119] self: [3:7571518379601663989:0] peer: [1:7571518376372020569:0] socket: 102 qp: -1
2025-11-11T17:18:17.822241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518377899868304:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.817922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518379601663993:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:17.817978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518379601663960:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.818006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518379601663960:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.825861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.826920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518375306696597:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(29035371ns) {HandleHandshakeStatus:329(29031512ns) {DropIncomingHandshake:441(1831ns) DropOutgoingHandshake:456(209ns) SwitchToState:178(442ns) ProcessPendingSessionEvents:495(789ns)}}
2025-11-11T17:18:17.813679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518380288826774:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.813718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518380288826774:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:17.821806Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518380288826774:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.809224Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518377646846636:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:17.834236Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518379601663960:2102] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:17.822720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518380288826733:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.809603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518373351879277:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:17.823982Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.824998Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518373351879277:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518377646846636:2120] held: no
2025-11-11T17:18:17.827705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846645:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.828053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846646:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.828137Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518377646846636:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:17.833783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518373351879277:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.834452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:17.834940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518373351879277:2065] [node 4] ICP22 created new session: [6:7571518377646846647:2065]
2025-11-11T17:18:17.835190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518377646846647:2065] [node 4] ICS09 handshake done sender: [6:7571518377646846636:2120] self: [6:7571518377646846644:0] peer: [4:7571518380288826773:0] socket: 79 qp: -1
2025-11-11T17:18:17.835228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518377646846647:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:17.835262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:17.822771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:17.822814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518380288826733:2077] [node 6] ICP22 created new session: [4:7571518380288826799:2077]
2025-11-11T17:18:17.822852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518380288826799:2077] [node 6] ICS09 handshake done sender: [4:7571518380288826774:2117] self: [4:7571518380288826773:0] peer: [6:7571518377646846644:0] socket: 95 qp: -1
2025-11-11T17:18:17.822885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518380288826799:2077] [node 6] ICS10 traffic start
2025-11-11T17:18:17.822924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:17.836481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518379601663944:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.836567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:17.836608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518379601663944:2068] [node 6] ICP22 created new session: [3:7571518379601663996:2068]
2025-11-11T17:18:17.836647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518379601663996:2068] [node 6] ICS09 handshake done sender: [3:7571518379601663960:2102] self: [3:7571518379601663959:0] peer: [6:7571518377646846642:0] socket: 112 qp: -1
2025-11-11T17:18:17.836670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518379601663996:2068] [node 6] ICS10 traffic start
2025-11-11T17:18:17.836702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:17.837974Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518377899868241:2073] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:17.838324Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16792) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.841993Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518377646846633:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:17.842338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518373351879255:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16792) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518377646846633:2119] held: no
2025-11-11T17:18:17.842419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518373351879255:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.842460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:17.842500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518373351879255:2050] [node 3] ICP22 created new session: [6:7571518377646846650:2050]
2025-11-11T17:18:17.842540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518377646846650:2050] [node 3] ICS09 handshake done sender: [6:7571518377646846633:2119] self: [6:7571518377646846642:0] peer: [3:7571518379601663959:0] socket: 72 qp: -1
2025-11-11T17:18:17.842567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518377646846650:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:17.842595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:17.842988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518377646846649:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16174
2025-11-11T17:18:17.846014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518373604900939:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:17.846073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:17.846121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518373604900939:2063] [node 6] ICP22 created new session: [5:7571518377899868307:2063]
2025-11-11T17:18:17.846170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518377899868307:2063] [node 6] ICS09 handshake done sender: [5:7571518377899868241:2073] self: [5:7571518377899868240:0] peer: [6:7571518377646846643:0] socket: 68 qp: -1
2025-11-11T17:18:17.846199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518377899868307:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:17.846224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:17.874049Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:17.875111Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518377646846637:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:17.875397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518373351879278:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518377646846637:2121] held: no
2025-11-11T17:18:17.875462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518373351879278:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:17.875522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:17.875566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518373351879278:2067] [node 5] ICP22 created new session: [6:7571518377646846652:2067]
2025-11-11T17:18:17.875603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518377646846652:2067] [node 5] ICS09 handshake done sender: [6:7571518377646846637:2121] self: [6:7571518377646846643:0] peer: [5:7571518377899868240:0] socket: 111 qp: -1
2025-11-11T17:18:17.875634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518377646846652:2067] [node 5] ICS10 traffic start
2025-11-11T17:18:17.875668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:17.909727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:17.909938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518376512911664:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-11-11T17:18:17.909973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518372217944279:2048] [node 1] ICP30 unregister session Session# [2:7571518376512911664:2048] VirtualId# [2:7571518376512911658:0]
2025-11-11T17:18:17.910004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518376512911664:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.910090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:17.910129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:17.913714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:17.913912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518379601663993:2048] [node 1] ICS01 socket: 102 reason# EndOfStream
2025-11-11T17:18:17.913978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518375306696597:2048] [node 1] ICP30 unregister session Session# [3:7571518379601663993:2048] VirtualId# [3:7571518379601663989:0]
2025-11-11T17:18:17.914011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518379601663993:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.914084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:17.914118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:17.917660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:17.917810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518380288826792:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-11T17:18:17.917836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518375993859400:2048] [node 1] ICP30 unregister session Session# [4:7571518380288826792:2048] VirtualId# [4:7571518380288826781:0]
2025-11-11T17:18:17.917856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518380288826792:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.917904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:17.917931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:17.917472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:17.917696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518377646846634:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-11-11T17:18:17.917730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518373351879253:2048] [node 1] ICP30 unregister session Session# [6:7571518377646846634:2048] VirtualId# [6:7571518377646846629:0]
2025-11-11T17:18:17.917757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518377646846634:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.917825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:17.917861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:17.917935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:17.918079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518377899868302:2048] [node 1] ICS01 socket: 129 reason# EndOfStream
2025-11-11T17:18:17.918106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518373604900909:2048] [node 1] ICP30 unregister session Session# [5:7571518377899868302:2048] VirtualId# [5:7571518377899868300:0]
2025-11-11T17:18:17.918123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518377899868302:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.918163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:17.918193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:17.973795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:17.974011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518379601663982:2049] [node 2] ICS01 socket: 83 reason# EndOfStream
2025-11-11T17:18:17.974055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518375306696598:2049] [node 2] ICP30 unregister session Session# [3:7571518379601663982:2049] VirtualId# [3:7571518379601663961:0]
2025-11-11T17:18:17.974130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518379601663982:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.974214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:17.974257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:17.978028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:17.978232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518377899868293:2049] [node 2] ICS01 socket: 101 reason# EndOfStream
2025-11-11T17:18:17.978267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518373604900910:2049] [node 2] ICP30 unregister session Session# [5:7571518377899868293:2049] VirtualId# [5:7571518377899868286:0]
2025-11-11T17:18:17.978295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518377899868293:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.978365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:17.978409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:17.978487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:17.978642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518377646846638:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-11-11T17:18:17.978673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518373351879254:2049] [node 2] ICP30 unregister session Session# [6:7571518377646846638:2049] VirtualId# [6:7571518377646846630:0]
2025-11-11T17:18:17.978692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518377646846638:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.978734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:17.978762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:17.981783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:17.982061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518380288826789:2049] [node 2] ICS01 socket: 92 reason# EndOfStream
2025-11-11T17:18:17.982108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518375993859401:2049] [node 2] ICP30 unregister session Session# [4:7571518380288826789:2049] VirtualId# [4:7571518380288826779:0]
2025-11-11T17:18:17.982163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518380288826789:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:17.982265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:17.982328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:18.043537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:18.043756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518377899868298:2050] [node 3] ICS01 socket: 91 reason# EndOfStream
2025-11-11T17:18:18.043789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518373604900911:2050] [node 3] ICP30 unregister session Session# [5:7571518377899868298:2050] VirtualId# [5:7571518377899868288:0]
2025-11-11T17:18:18.043819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518377899868298:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:18.043900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:18.043953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:18.044280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:18.044489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518377646846650:2050] [node 3] ICS01 socket: 72 reason# EndOfStream
2025-11-11T17:18:18.044519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518373351879255:2050] [node 3] ICP30 unregister session Session# [6:7571518377646846650:2050] VirtualId# [6:7571518377646846642:0]
2025-11-11T17:18:18.044540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518377646846650:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:18.044582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:18.044609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:18.044782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:18.044946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518380288826794:2050] [node 3] ICS01 socket: 94 reason# EndOfStream
2025-11-11T17:18:18.044970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518375993859402:2050] [node 3] ICP30 unregister session Session# [4:7571518380288826794:2050] VirtualId# [4:7571518380288826783:0]
2025-11-11T17:18:18.044991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518380288826794:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:18.045043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:18.045072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:18.089805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:18.090146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518377646846647:2065] [node 4] ICS01 socket: 79 reason# EndOfStream
2025-11-11T17:18:18.090314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:18.090576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518377899868296:2059] [node 4] ICS01 socket: 104 reason# EndOfStream
2025-11-11T17:18:18.090636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518373604900922:2059] [node 4] ICP30 unregister session Session# [5:7571518377899868296:2059] VirtualId# [5:7571518377899868287:0]
2025-11-11T17:18:18.090693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518377899868296:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:18.090817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:18.090917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:18.091630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518373351879277:2065] [node 4] ICP30 unregister session Session# [6:7571518377646846647:2065] VirtualId# [6:7571518377646846644:0]
2025-11-11T17:18:18.091697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518377646846647:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:18.091798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:18.091863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:18.169751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:18.169979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518377646846652:2067] [node 5] ICS01 socket: 111 reason# EndOfStream
2025-11-11T17:18:18.170018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518373351879278:2067] [node 5] ICP30 unregister session Session# [6:7571518377646846652:2067] VirtualId# [6:7571518377646846643:0]
2025-11-11T17:18:18.170051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518377646846652:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:18.170136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:18.170175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:18.299635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518384848838623:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:18.299673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518384848838624:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:18.357903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518383770430719:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:18.357952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518383770430720:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:18.413903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518380706775380:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:18.413937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518380706775381:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:18.453966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518383850266729:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:18.454004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518383850266730:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:18.454024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518383850266731:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:18.501608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518382329845821:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:18.501658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518382329845822:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:18.501681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518382329845823:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:18.577892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518384042841810:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:18.577928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518384042841811:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:18.577952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518384042841812:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:18.848102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518383850266747:2064] [node 5] ICP01 ready to work
2025-11-11T17:18:18.844245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518380706775393:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-11-11T17:18:18.855657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518384042841827:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:18.861682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518380706775392:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-11-11T17:18:18.953921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518380706775395:2061] [node 5] ICP01 ready to work
2025-11-11T17:18:18.969789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518384848838686:2097] [node 5] ICP01 ready to work
2025-11-11T17:18:19.033828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518385001742744:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:19.033639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518388065398081:2079] [node 5] ICP01 ready to work
2025-11-11T17:18:19.061658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518383770430782:2077] [node 4] ICP01 ready to work
2025-11-11T17:18:19.110191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518384042841811:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(93500437ns) {Configure:104(93497889ns) {ProcessConfigured:135(61152ns) {StartInitialHandshake:157(57762ns) {DropHandshakes:469(3241ns) {DropIncomingHandshake:441(703ns) DropOutgoingHandshake:456(616ns)} GenerateSessionVirtualId:988(1604ns) PrepareNewSessionHandshake:476(1466ns) {SwitchToState:178(493ns)}}}}}
2025-11-11T17:18:18.987608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518384848838687:2099] [node 6] ICP01 ready to work
2025-11-11T17:18:19.137651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518388145234088:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.178065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518385001742748:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:19.174543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518388065398081:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(64442924ns) {Configure:104(64436366ns) {ProcessConfigured:135(84492ns) {StartInitialHandshake:157(82338ns) {DropHandshakes:469(6358ns) {DropIncomingHandshake:441(826ns) DropOutgoingHandshake:456(566ns)} GenerateSessionVirtualId:988(1433ns) PrepareNewSessionHandshake:476(2190ns) {SwitchToState:178(734ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.190487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518388065398088:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.202111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518388337809177:2064] [node 5] ICP01 ready to work
2025-11-11T17:18:19.204469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518386624813179:2061] [node 4] ICP01 ready to work
2025-11-11T17:18:19.143200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518384848838623:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(59316770ns) {Configure:104(59308589ns) {ProcessConfigured:135(68549ns) {StartInitialHandshake:157(66458ns) {DropHandshakes:469(5195ns) {DropIncomingHandshake:441(445ns) DropOutgoingHandshake:456(473ns)} GenerateSessionVirtualId:988(660ns) PrepareNewSessionHandshake:476(1795ns) {SwitchToState:178(591ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.173737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518389143805984:2101] [node 4] ICP01 ready to work
2025-11-11T17:18:19.218924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518382329845821:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(295537391ns) {Configure:104(295533598ns) {ProcessConfigured:135(70530ns) {StartInitialHandshake:157(67911ns) {DropHandshakes:469(4070ns) {DropIncomingHandshake:441(999ns) DropOutgoingHandshake:456(588ns)} GenerateSessionVirtualId:988(1082ns) PrepareNewSessionHandshake:476(2229ns) {SwitchToState:178(750ns)}}}}}
2025-11-11T17:18:19.219411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518386624813184:2074] [node 6] ICP01 ready to work
2025-11-11T17:18:19.225451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813180:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.251181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518386624813184:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(30192224ns) {Configure:104(30188320ns) {ProcessConfigured:135(13881532ns) {StartInitialHandshake:157(13879124ns) {DropHandshakes:469(684ns) {DropIncomingHandshake:441(137ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(161ns) PrepareNewSessionHandshake:476(6245ns) {SwitchToState:178(3067ns)}}}}}
2025-11-11T17:18:19.251324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813188:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.251513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813189:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.270731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813187:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.252172Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518380706775393:2060] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.290099Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518386624813188:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.203917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809178:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.211697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518384848838624:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(151372564ns) {Configure:104(151363362ns) {ProcessConfigured:135(81383ns) {StartInitialHandshake:157(79009ns) {DropHandshakes:469(2665ns) {DropIncomingHandshake:441(587ns) DropOutgoingHandshake:456(775ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:476(1393ns) {SwitchToState:178(507ns)}}}}}
2025-11-11T17:18:19.278332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518389143805996:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38735
2025-11-11T17:18:19.278490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518389143805997:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38735
2025-11-11T17:18:19.278543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518389143805998:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38735
2025-11-11T17:18:19.278590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518389143805999:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38735
2025-11-11T17:18:19.282460Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805996:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.301709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518388337809178:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.293927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518383850266730:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(331350256ns) {Configure:104(331345658ns) {ProcessConfigured:135(12689510ns) {StartInitialHandshake:157(12684677ns) {DropHandshakes:469(3450ns) {DropIncomingHandshake:441(842ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:476(16768ns) {SwitchToState:178(3246ns)}}}}}
2025-11-11T17:18:19.299110Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518380706775392:2059] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.330686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809185:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.329787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805990:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.301715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518388065398090:2094] [node 6] ICP01 ready to work
2025-11-11T17:18:19.313726Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518388065398088:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.301930Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518386624813189:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.329818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518389143805990:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.337966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518384042841810:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.353724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518383770430719:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.333877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518386624813187:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.358164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518384042841810:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(26854939ns) {IncomingHandshake:261(26851801ns) {DropIncomingHandshake:441(403ns) PrepareNewSessionHandshake:476(1880ns) {SwitchToState:178(740ns)} GenerateSessionVirtualId:988(792ns) IssueIncomingHandshakeReply:193(12269757ns) {SwitchToState:178(2242ns)}}}
2025-11-11T17:18:19.369715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518384042841827:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(432236484ns) {Configure:104(432221997ns) {ProcessConfigured:135(32250313ns) {StartInitialHandshake:157(32247393ns) {DropHandshakes:469(3655ns) {DropIncomingHandshake:441(834ns) DropOutgoingHandshake:456(516ns)} GenerateSessionVirtualId:988(1567ns) PrepareNewSessionHandshake:476(2335ns) {SwitchToState:178(773ns)}}}}}
2025-11-11T17:18:19.382417Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518388337809178:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:19.386255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809187:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.355225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518383850266731:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(352711194ns) {Configure:104(321274112ns) {ProcessConfigured:135(54669ns) {StartInitialHandshake:157(52821ns) {DropHandshakes:469(4391ns) {DropIncomingHandshake:441(530ns) DropOutgoingHandshake:456(297ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:476(1183ns) {SwitchToState:178(429ns)}}}}}
2025-11-11T17:18:19.330969Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518389143805990:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:19.356244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518385001742744:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(246357496ns) {Configure:104(223029792ns) {ProcessConfigured:135(12548192ns) {StartInitialHandshake:157(12546560ns) {DropHandshakes:469(2907ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(497ns)} GenerateSessionVirtualId:988(1093ns) PrepareNewSessionHandshake:476(2711ns) {SwitchToState:178(1215ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.355587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234097:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
2025-11-11T17:18:19.331535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518384848838687:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.394114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518382329845821:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.361864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518380706775381:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(399732806ns) {Configure:104(399728938ns) {ProcessConfigured:135(29484472ns) {StartInitialHandshake:157(29481491ns) {DropHandshakes:469(2740ns) {DropIncomingHandshake:441(947ns) DropOutgoingHandshake:456(348ns)} GenerateSessionVirtualId:988(713ns) PrepareNewSessionHandshake:476(4229ns) {SwitchToState:178(2159ns)}}}}}
2025-11-11T17:18:19.414850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518388065398090:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(75212529ns) {Configure:104(75202698ns) {ProcessConfigured:135(65255ns) {StartInitialHandshake:157(63119ns) {DropHandshakes:469(2434ns) {DropIncomingHandshake:441(482ns) DropOutgoingHandshake:456(379ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:476(1920ns) {SwitchToState:178(869ns)}}}}}
2025-11-11T17:18:19.402061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813199:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.361959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518385001742754:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-11-11T17:18:19.362019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518385001742755:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-11-11T17:18:19.362226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518380706775381:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.402541Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518385001742755:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.434133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518380706775380:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.444680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518385001742759:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-11-11T17:18:19.331571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:19.417062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518384042841810:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.402215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518388145234097:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.331609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518384848838687:2099] [node 6] ICP22 created new session: [1:7571518389143806002:2099]
2025-11-11T17:18:19.331648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518389143806002:2099] [node 6] ICS09 handshake done sender: [1:7571518389143805990:2112] self: [1:7571518389143805989:0] peer: [6:7571518388337809184:0] socket: 86 qp: -1
2025-11-11T17:18:19.331676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518389143806002:2099] [node 6] ICS10 traffic start
2025-11-11T17:18:19.331701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:19.358114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805999:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.400723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805988:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.400748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518389143805988:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.402447Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805998:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.438112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518389143805984:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(230921891ns) {Configure:104(230915938ns) {ProcessConfigured:135(61276ns) {StartInitialHandshake:157(59237ns) {DropHandshakes:469(4817ns) {DropIncomingHandshake:441(588ns) DropOutgoingHandshake:456(515ns)} GenerateSessionVirtualId:988(656ns) PrepareNewSessionHandshake:476(1309ns) {SwitchToState:178(546ns)}}}}}
2025-11-11T17:18:19.442154Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518389143805988:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:19.453293Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805986:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.453330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518389143805986:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.458198Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805995:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.458235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518389143805995:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.462525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518384848838687:2099] [node 6] ICP18 handshake (actor [1:7571518389143805998:2116]) is too old
2025-11-11T17:18:19.415180Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518383770430749:2076] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.411098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518386624813180:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.417129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:19.415197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518383770430749:2076] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.426190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518384042841810:2048] [node 1] ICP22 created new session: [6:7571518388337809188:2048]
2025-11-11T17:18:19.445917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518388337809188:2048] [node 1] ICS09 handshake done sender: [6:7571518388337809178:2113] self: [6:7571518388337809184:0] peer: [1:7571518389143805989:0] socket: 91 qp: -1
2025-11-11T17:18:19.453111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518388337809188:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:19.458168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:19.493664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518388337809187:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.464467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518380706775380:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17318710ns) {IncomingHandshake:261(17316656ns) {DropIncomingHandshake:441(1275ns) PrepareNewSessionHandshake:476(2970ns) {SwitchToState:178(2058ns)} GenerateSessionVirtualId:988(778ns) IssueIncomingHandshakeReply:193(8097665ns) {SwitchToState:178(1133ns)}}}
2025-11-11T17:18:19.480190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518384848838686:2097] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.511526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809191:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.480313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:19.480352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518384848838686:2097] [node 5] ICP22 created new session: [1:7571518389143806010:2097]
2025-11-11T17:18:19.480400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518389143806010:2097] [node 5] ICS09 handshake done sender: [1:7571518389143805988:2111] self: [1:7571518389143805987:0] peer: [5:7571518386624813198:0] socket: 85 qp: -1
2025-11-11T17:18:19.480428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518389143806010:2097] [node 5] ICS10 traffic start
2025-11-11T17:18:19.480461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:19.482050Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518389143805986:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:19.509903Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143805997:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.530081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518384042841810:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(118631994ns) {HandleHandshakeStatus:329(118630087ns) {DropIncomingHandshake:441(3115ns) DropOutgoingHandshake:456(6136ns) SwitchToState:178(745ns) ProcessPendingSessionEvents:495(20101715ns) {ForwardSessionEventToSession:612(20046055ns) ForwardSessionEventToSession:612(8966ns) ForwardSessionEventToSession:612(2333ns) ForwardSessionEventToSession:612(2263ns) ForwardSessionEventToSession:612(6756ns) ForwardSessionEventToSession:612(1980ns) ForwardSessionEventToSession:612(4735ns) ForwardSessionEventToSession:612(1869ns) ForwardSessionEventToSession:612(12274ns) ForwardSessionEventToSession:612(4050ns)}}}
2025-11-11T17:18:19.446228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813201:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.490290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518383850266729:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(275216952ns) {Configure:104(275206780ns) {ProcessConfigured:135(15248160ns) {StartInitialHandshake:157(15246594ns) {DropHandshakes:469(2125ns) {DropIncomingHandshake:441(550ns) DropOutgoingHandshake:456(425ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:476(2897ns) {SwitchToState:178(1455ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.469848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518385001742754:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.463580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518382329845822:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(211039563ns) {Configure:104(211034502ns) {ProcessConfigured:135(13615716ns) {StartInitialHandshake:157(13613452ns) {DropHandshakes:469(2017ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(526ns)} GenerateSessionVirtualId:988(586ns) PrepareNewSessionHandshake:476(4028ns) {SwitchToState:178(1470ns)}}}}}
2025-11-11T17:18:19.545791Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518389143805995:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:19.541717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518384042841827:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.416567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518383770430719:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32042160ns) {IncomingHandshake:261(12072546ns) {DropIncomingHandshake:441(998ns) PrepareNewSessionHandshake:476(2291ns) {SwitchToState:178(1573ns)} GenerateSessionVirtualId:988(797ns) IssueIncomingHandshakeReply:193(8108804ns) {SwitchToState:178(2123ns)}}}
2025-11-11T17:18:19.486459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518385001742748:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(182311722ns) {Configure:104(182304794ns) {ProcessConfigured:135(55954ns) {StartInitialHandshake:157(53206ns) {DropHandshakes:469(1992ns) {DropIncomingHandshake:441(499ns) DropOutgoingHandshake:456(430ns)} GenerateSessionVirtualId:988(636ns) PrepareNewSessionHandshake:476(1038ns) {SwitchToState:178(370ns)}}}}}
2025-11-11T17:18:19.554678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518384848838623:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.463923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518382329845822:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.421880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518388065398094:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27
2025-11-11T17:18:19.493638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518383850266749:2065] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.487086Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518380706775392:2059] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:19.487373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518380706775381:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.487438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:19.487474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518380706775381:2049] [node 2] ICP22 created new session: [3:7571518385001742764:2049]
2025-11-11T17:18:19.487512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518385001742764:2049] [node 2] ICS09 handshake done sender: [3:7571518380706775392:2059] self: [3:7571518385001742757:0] peer: [2:7571518383770430748:0] socket: 79 qp: -1
2025-11-11T17:18:19.487541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518385001742764:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:19.487576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:19.518476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518385001742767:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-11-11T17:18:19.583817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518385001742746:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.583858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518385001742746:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.585873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518385001742746:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:19.585946Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518385001742755:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:19.614429Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:19.615367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518380706775380:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.554138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.615419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:19.615461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518380706775380:2048] [node 1] ICP22 created new session: [3:7571518385001742769:2048]
2025-11-11T17:18:19.615466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518380706775395:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.615498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518385001742769:2048] [node 1] ICS09 handshake done sender: [3:7571518385001742755:2117] self: [3:7571518385001742758:0] peer: [1:7571518389143805994:0] socket: 111 qp: -1
2025-11-11T17:18:19.615530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:19.615583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518380706775395:2061] [node 5] ICP22 created new session: [3:7571518385001742770:2061]
2025-11-11T17:18:19.615630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518385001742770:2061] [node 5] ICS09 handshake done sender: [3:7571518385001742746:2113] self: [3:7571518385001742745:0] peer: [5:7571518386624813212:0] socket: 112 qp: -1
2025-11-11T17:18:19.615661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518385001742770:2061] [node 5] ICS10 traffic start
2025-11-11T17:18:19.615689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:19.554731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:19.464390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518386624813179:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.526759Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518386624813187:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:19.493680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518383850266749:2065] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.631458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518385001742769:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:19.609891Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518388337809191:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.554764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518384848838623:2048] [node 2] ICP22 created new session: [1:7571518389143806015:2048]
2025-11-11T17:18:19.433764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518388065398094:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.433905Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518383770430749:2076] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:19.457636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518388065398095:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27
2025-11-11T17:18:19.497919Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518388065398087:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.497943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518388065398087:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.498827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518383770430720:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.498870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:19.498911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518383770430720:2049] [node 3] ICP22 created new session: [2:7571518388065398101:2049]
2025-11-11T17:18:19.498961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518388065398101:2049] [node 3] ICS09 handshake done sender: [2:7571518383770430749:2076] self: [2:7571518383770430748:0] peer: [3:7571518385001742757:0] socket: 78 qp: -1
2025-11-11T17:18:19.498992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518388065398101:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:19.499022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:19.499673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518388065398095:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.500293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518383770430720:2049] [node 3] ICP18 handshake (actor [2:7571518388065398095:2116]) is too old
2025-11-11T17:18:19.500402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518388065398099:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27
2025-11-11T17:18:19.502658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518388065398100:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27
2025-11-11T17:18:19.503310Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518388065398087:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:19.503553Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518388065398088:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:19.503733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518388065398081:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.503769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:19.503811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518388065398081:2079] [node 5] ICP22 created new session: [2:7571518388065398105:2079]
2025-11-11T17:18:19.503859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518388065398105:2079] [node 5] ICS09 handshake done sender: [2:7571518388065398087:2112] self: [2:7571518388065398086:0] peer: [5:7571518386624813204:0] socket: 89 qp: -1
2025-11-11T17:18:19.503886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518388065398105:2079] [node 5] ICS10 traffic start
2025-11-11T17:18:19.503914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:19.504244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518383770430719:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.504278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:19.504307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518383770430719:2048] [node 1] ICP22 created new session: [2:7571518388065398107:2048]
2025-11-11T17:18:19.504341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518388065398107:2048] [node 1] ICS09 handshake done sender: [2:7571518388065398088:2113] self: [2:7571518388065398091:0] peer: [1:7571518389143805985:0] socket: 88 qp: -1
2025-11-11T17:18:19.504370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518388065398107:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:19.504390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:19.521712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518383770430782:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(327440357ns) {Configure:104(327434617ns) {ProcessConfigured:135(34953363ns) {StartInitialHandshake:157(34950269ns) {DropHandshakes:469(2597ns) {DropIncomingHandshake:441(529ns) DropOutgoingHandshake:456(384ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:476(3329ns) {SwitchToState:178(1287ns)}}}}}
2025-11-11T17:18:19.589781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518388065398100:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.649763Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518388065398093:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.649794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518388065398093:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.642930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:19.530824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234106:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
2025-11-11T17:18:19.527063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518382329845821:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.554807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518389143806015:2048] [node 2] ICS09 handshake done sender: [1:7571518389143805986:2110] self: [1:7571518389143805985:0] peer: [2:7571518388065398091:0] socket: 84 qp: -1
2025-11-11T17:18:19.670406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518380706775380:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(54843514ns) {HandleHandshakeStatus:329(54841088ns) {DropIncomingHandshake:441(399ns) DropOutgoingHandshake:456(3717ns) SwitchToState:178(216ns) ProcessPendingSessionEvents:495(8475029ns) {ForwardSessionEventToSession:612(8412884ns) ForwardSessionEventToSession:612(10631ns) ForwardSessionEventToSession:612(3298ns) ForwardSessionEventToSession:612(2558ns) ForwardSessionEventToSession:612(16424ns) ForwardSessionEventToSession:612(3309ns) ForwardSessionEventToSession:612(7002ns) ForwardSessionEventToSession:612(3909ns) ForwardSessionEventToSession:612(2075ns) ForwardSessionEventToSession:612(3938ns)}}}
2025-11-11T17:18:19.670471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518380706775380:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.629830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518384042841811:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.659317Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518388065398093:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:19.527118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:19.530868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234107:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.685484Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518385001742752:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.554834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518389143806015:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:19.636093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518388337809192:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.685515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518385001742752:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.686437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518385001742774:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-11-11T17:18:19.701334Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518385001742752:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:19.684180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518388065398090:2094] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.527160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518382329845821:2048] [node 1] ICP22 created new session: [5:7571518386624813207:2048]
2025-11-11T17:18:19.636240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518388337809177:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.670752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809196:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.685265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809198:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.686554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809199:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.687044Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518388337809192:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:19.699721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518388337809177:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.699763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:19.699801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518388337809177:2064] [node 5] ICP22 created new session: [6:7571518388337809200:2064]
2025-11-11T17:18:19.699859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518388337809200:2064] [node 5] ICS09 handshake done sender: [6:7571518388337809192:2119] self: [6:7571518388337809195:0] peer: [5:7571518386624813193:0] socket: 132 qp: -1
2025-11-11T17:18:19.699908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518388337809200:2064] [node 5] ICS10 traffic start
2025-11-11T17:18:19.699941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:19.700363Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518388337809196:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.700440Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:19.713797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518388337809191:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:19.722268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518384042841811:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518388337809191:2118] held: no
2025-11-11T17:18:19.722410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809197:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.722780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518384042841811:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.722842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:19.722881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518384042841811:2049] [node 2] ICP22 created new session: [6:7571518388337809202:2049]
2025-11-11T17:18:19.722923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518388337809202:2049] [node 2] ICS09 handshake done sender: [6:7571518388337809191:2118] self: [6:7571518388337809194:0] peer: [2:7571518388065398092:0] socket: 126 qp: -1
2025-11-11T17:18:19.722951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518388337809202:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:19.722979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:19.723315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518384042841812:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.550019Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518383850266749:2065] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:19.738499Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518385001742774:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.738558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518385001742744:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.738642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:19.738689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518385001742744:2070] [node 4] ICP22 created new session: [3:7571518385001742776:2070]
2025-11-11T17:18:19.738730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518385001742776:2070] [node 4] ICS09 handshake done sender: [3:7571518385001742752:2115] self: [3:7571518385001742751:0] peer: [4:7571518388145234116:0] socket: 100 qp: -1
2025-11-11T17:18:19.738763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518385001742776:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:19.738799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.684230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:19.554864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.630202Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518388145234107:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.749731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518380706775395:2061] [node 5] ICP35 event processing took too much time PendingConnection:278(66359688ns) {HandleHandshakeStatus:329(66356801ns) {DropIncomingHandshake:441(15557ns) DropOutgoingHandshake:456(303ns) SwitchToState:178(352ns) ProcessPendingSessionEvents:495(65992861ns) {ForwardSessionEventToSession:612(6218582ns) ForwardSessionEventToSession:612(19987280ns) ForwardSessionEventToSession:612(11795583ns) ForwardSessionEventToSession:612(5569401ns) ForwardSessionEventToSession:612(4017ns) ForwardSessionEventToSession:612(2199ns) ForwardSessionEventToSession:612(7329135ns) ForwardSessionEventToSession:612(3389ns) ForwardSessionEventToSession:612(2280ns) ForwardSessionEventToSession:612(15046578ns)}}}
2025-11-11T17:18:19.753846Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518385001742761:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.753878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518385001742761:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.633817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518388145234096:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.633849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518388145234096:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.653932Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518388145234106:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.761805Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518385001742761:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:19.755004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518388337809206:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38726
2025-11-11T17:18:19.555225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518384848838624:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.527206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518386624813207:2048] [node 1] ICS09 handshake done sender: [5:7571518386624813187:2113] self: [5:7571518386624813198:0] peer: [1:7571518389143805987:0] socket: 94 qp: -1
2025-11-11T17:18:19.684265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518388065398090:2094] [node 6] ICP22 created new session: [2:7571518388065398113:2094]
2025-11-11T17:18:19.654339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518383850266729:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.777790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518385001742744:2070] [node 4] ICP18 handshake (actor [3:7571518385001742774:2121]) is too old
2025-11-11T17:18:19.555255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:19.755537Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518388337809187:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:19.762469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518384042841827:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.762516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:19.762564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518384042841827:2059] [node 4] ICP22 created new session: [6:7571518388337809207:2059]
2025-11-11T17:18:19.762607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518388337809207:2059] [node 4] ICS09 handshake done sender: [6:7571518388337809187:2117] self: [6:7571518388337809193:0] peer: [4:7571518388145234095:0] socket: 113 qp: -1
2025-11-11T17:18:19.762653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518388337809207:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:19.762684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:19.774592Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518388337809196:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:19.684306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518388065398113:2094] [node 6] ICS09 handshake done sender: [2:7571518388065398093:2114] self: [2:7571518388065398092:0] peer: [6:7571518388337809194:0] socket: 117 qp: -1
2025-11-11T17:18:19.527239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518386624813207:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:19.778363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518385001742748:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.555290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518384848838624:2049] [node 3] ICP22 created new session: [1:7571518389143806017:2049]
2025-11-11T17:18:19.654399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518383850266747:2064] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:19.527270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:19.654458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:19.654505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518383850266747:2064] [node 5] ICP22 created new session: [4:7571518388145234110:2064]
2025-11-11T17:18:19.654567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518388145234110:2064] [node 5] ICS09 handshake done sender: [4:7571518383850266749:2065] self: [4:7571518383850266748:0] peer: [5:7571518386624813200:0] socket: 82 qp: -1
2025-11-11T17:18:19.654604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518388145234110:2064] [node 5] ICS10 traffic start
2025-11-11T17:18:19.654646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:19.656013Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518388145234096:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:19.682536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518388145234088:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.682578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:19.682614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518388145234088:2063] [node 6] ICP22 created new session: [4:7571518388145234113:2063]
2025-11-11T17:18:19.682654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518388145234113:2063] [node 6] ICS09 handshake done sender: [4:7571518388145234096:2112] self: [4:7571518388145234095:0] peer: [6:7571518388337809193:0] socket: 109 qp: -1
2025-11-11T17:18:19.682688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518388145234113:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:19.682715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:19.683241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518383850266731:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.683368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234112:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
2025-11-11T17:18:19.689519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234118:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
2025-11-11T17:18:19.699492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234119:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
2025-11-11T17:18:19.707317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518383850266747:2064] [node 5] ICP35 event processing took too much time PendingConnection:278(52720705ns) {HandleHandshakeStatus:329(52715929ns) {DropIncomingHandshake:441(11879ns) DropOutgoingHandshake:456(483ns) SwitchToState:178(164ns) ProcessPendingSessionEvents:495(25392595ns) {ForwardSessionEventToSession:612(25338704ns) ForwardSessionEventToSession:612(8060ns) ForwardSessionEventToSession:612(2479ns) ForwardSessionEventToSession:612(8029ns) ForwardSessionEventToSession:612(3071ns) ForwardSessionEventToSession:612(6835ns) ForwardSessionEventToSession:612(3423ns) ForwardSessionEventToSession:612(1945ns) ForwardSessionEventToSession:612(5258ns) ForwardSessionEventToSession:612(4892ns)}}}
2025-11-11T17:18:19.709786Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518388145234107:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:19.724097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518383850266729:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.724156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:19.724190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518383850266729:2048] [node 1] ICP22 created new session: [4:7571518388145234120:2048]
2025-11-11T17:18:19.724219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518388145234120:2048] [node 1] ICS09 handshake done sender: [4:7571518388145234107:2117] self: [4:7571518388145234109:0] peer: [1:7571518389143806006:0] socket: 133 qp: -1
2025-11-11T17:18:19.724240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518388145234120:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:19.724283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:19.738099Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518388145234112:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.738235Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518388145234106:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:19.746281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234122:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
2025-11-11T17:18:19.746425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518383850266730:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.758354Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518388145234122:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.764081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518383850266731:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.765413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518388145234088:2063] [node 6] ICP18 handshake (actor [4:7571518388145234122:2121]) is too old
2025-11-11T17:18:19.765569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:19.774960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518383850266731:2050] [node 3] ICP22 created new session: [4:7571518388145234124:2050]
2025-11-11T17:18:19.789995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518388145234124:2050] [node 3] ICS09 handshake done sender: [4:7571518388145234106:2116] self: [4:7571518388145234116:0] peer: [3:7571518385001742751:0] socket: 87 qp: -1
2025-11-11T17:18:19.790042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518388145234124:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:19.790081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:19.791217Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518383850266731:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(32731671ns) {HandleHandshakeStatus:329(32727740ns) {DropIncomingHandshake:441(1599ns) DropOutgoingHandshake:456(5662ns) SwitchToState:178(336ns) ProcessPendingSessionEvents:495(85452ns) {ForwardSessionEventToSession:612(44918ns) ForwardSessionEventToSession:612(2458ns) ForwardSessionEventToSession:612(2874ns) ForwardSessionEventToSession:612(8484ns) ForwardSessionEventToSession:612(5924ns) ForwardSessionEventToSession:612(3861ns) ForwardSessionEventToSession:612(1823ns) ForwardSessionEventToSession:612(1695ns) ForwardSessionEventToSession:612(4172ns) ForwardSessionEventToSession:612(1745ns)}}}
2025-11-11T17:18:19.796934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518388145234126:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52835
2025-11-11T17:18:19.684333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518388065398113:2094] [node 6] ICS10 traffic start
2025-11-11T17:18:19.790616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518384042841812:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.555316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518389143806017:2049] [node 3] ICS09 handshake done sender: [1:7571518389143805995:2113] self: [1:7571518389143805994:0] peer: [3:7571518385001742758:0] socket: 107 qp: -1
2025-11-11T17:18:19.684354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:19.797151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:19.799875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518384042841812:2050] [node 3] ICP22 created new session: [6:7571518388337809209:2050]
2025-11-11T17:18:19.808094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518388337809209:2050] [node 3] ICS09 handshake done sender: [6:7571518388337809196:2120] self: [6:7571518388337809204:0] peer: [3:7571518385001742760:0] socket: 102 qp: -1
2025-11-11T17:18:19.821705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518388337809209:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:19.821760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:19.527856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813205:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.798290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:19.813714Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518388145234112:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:19.555334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518389143806017:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:19.798348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518385001742748:2066] [node 6] ICP22 created new session: [3:7571518385001742779:2066]
2025-11-11T17:18:19.798396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518385001742779:2066] [node 6] ICS09 handshake done sender: [3:7571518385001742761:2119] self: [3:7571518385001742760:0] peer: [6:7571518388337809204:0] socket: 90 qp: -1
2025-11-11T17:18:19.798433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518385001742779:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:19.798473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:19.798854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518385001742748:2066] [node 6] ICP35 event processing took too much time PendingConnection:278(20806672ns) {HandleHandshakeStatus:329(20331003ns) {DropIncomingHandshake:441(13859ns) DropOutgoingHandshake:456(4295ns) SwitchToState:178(1128ns) ProcessPendingSessionEvents:495(89046ns) {ForwardSessionEventToSession:612(42039ns) ForwardSessionEventToSession:612(4771ns) ForwardSessionEventToSession:612(6313ns) ForwardSessionEventToSession:612(1948ns) ForwardSessionEventToSession:612(2392ns) ForwardSessionEventToSession:612(4272ns) ForwardSessionEventToSession:612(2373ns) ForwardSessionEventToSession:612(3974ns) ForwardSessionEventToSession:612(2441ns) ForwardSessionEventToSession:612(10264ns)}}}
2025-11-11T17:18:19.528290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813206:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.555359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:19.681737Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143806007:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.681786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518389143806007:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.698019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518389143806019:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38735
2025-11-11T17:18:19.699080Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518389143806007:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:19.705789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518389143805984:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.705862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:19.705901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518389143805984:2101] [node 4] ICP22 created new session: [1:7571518389143806021:2101]
2025-11-11T17:18:19.705946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518389143806021:2101] [node 4] ICS09 handshake done sender: [1:7571518389143806007:2118] self: [1:7571518389143806006:0] peer: [4:7571518388145234109:0] socket: 122 qp: -1
2025-11-11T17:18:19.705978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518389143806021:2101] [node 4] ICS10 traffic start
2025-11-11T17:18:19.706014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:19.706453Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518389143806019:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.706625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518389143805984:2101] [node 4] ICP18 handshake (actor [1:7571518389143806019:2119]) is too old
2025-11-11T17:18:19.769710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518388065398098:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.528447Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518386624813188:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:19.528657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518382329845822:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.528693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:19.528734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518382329845822:2049] [node 2] ICP22 created new session: [5:7571518386624813209:2049]
2025-11-11T17:18:19.528770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518386624813209:2049] [node 2] ICS09 handshake done sender: [5:7571518386624813188:2114] self: [5:7571518386624813204:0] peer: [2:7571518388065398086:0] socket: 95 qp: -1
2025-11-11T17:18:19.528797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518386624813209:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:19.528821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:19.533443Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518386624813179:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(19238614ns) {IncomingHandshake:261(19236590ns) {DropIncomingHandshake:441(470ns) PrepareNewSessionHandshake:476(1615ns) {SwitchToState:178(743ns)} GenerateSessionVirtualId:988(426ns) IssueIncomingHandshakeReply:193(19218922ns) {SwitchToState:178(1775ns)}}}
2025-11-11T17:18:19.540215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518386624813199:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.549888Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:19.557075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518382329845821:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:19.570932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518382329845823:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:19.571120Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518386624813180:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:19.571345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518386624813179:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.571382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:19.571422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518386624813179:2061] [node 4] ICP22 created new session: [5:7571518386624813213:2061]
2025-11-11T17:18:19.571479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518386624813213:2061] [node 4] ICS09 handshake done sender: [5:7571518386624813180:2110] self: [5:7571518386624813200:0] peer: [4:7571518383850266748:0] socket: 83 qp: -1
2025-11-11T17:18:19.571512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518386624813213:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:19.571542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:19.599477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518386624813215:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20799
2025-11-11T17:18:19.635161Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:19.636183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518382329845823:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518386624813199:2118] held: no
2025-11-11T17:18:19.654782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518386624813194:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:19.654816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518386624813194:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.659248Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518386624813199:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:19.681963Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518386624813194:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:19.697721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518382329845823:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.697789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:19.705723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518382329845823:2050] [node 3] ICP22 created new session: [5:7571518386624813217:2050]
2025-11-11T17:18:19.725958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518386624813217:2050] [node 3] ICS09 handshake done sender: [5:7571518386624813199:2118] self: [5:7571518386624813212:0] peer: [3:7571518385001742745:0] socket: 115 qp: -1
2025-11-11T17:18:19.726078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518386624813184:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.739767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518386624813217:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:19.755417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:19.764304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:19.764351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518386624813184:2074] [node 6] ICP22 created new session: [5:7571518386624813219:2074]
2025-11-11T17:18:19.764412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518386624813219:2074] [node 6] ICS09 handshake done sender: [5:7571518386624813194:2117] self: [5:7571518386624813193:0] peer: [6:7571518388337809195:0] socket: 131 qp: -1
2025-11-11T17:18:19.764443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518386624813219:2074] [node 6] ICS10 traffic start
2025-11-11T17:18:19.764505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:19.764885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518386624813184:2074] [node 6] ICP35 event processing took too much time PendingConnection:278(63266646ns) {HandleHandshakeStatus:329(63264674ns) {DropIncomingHandshake:441(23482ns) DropOutgoingHandshake:456(238ns) SwitchToState:178(392ns) ProcessPendingSessionEvents:495(106344ns) {ForwardSessionEventToSession:612(51791ns) ForwardSessionEventToSession:612(3141ns) ForwardSessionEventToSession:612(3366ns) ForwardSessionEventToSession:612(5295ns) ForwardSessionEventToSession:612(15030ns) ForwardSessionEventToSession:612(1713ns) ForwardSessionEventToSession:612(5256ns) ForwardSessionEventToSession:612(1932ns) ForwardSessionEventToSession:612(10459ns) ForwardSessionEventToSession:612(1743ns)}}}
2025-11-11T17:18:19.781872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518382329845823:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(102629577ns) {HandleHandshakeStatus:329(102626876ns) {DropIncomingHandshake:441(2571ns) DropOutgoingHandshake:456(315ns) SwitchToState:178(594ns) ProcessPendingSessionEvents:495(1448ns)}}
2025-11-11T17:18:19.813990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518383850266730:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:19.850033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518384042841812:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(59203501ns) {HandleHandshakeStatus:329(59200172ns) {DropIncomingHandshake:441(1310ns) DropOutgoingHandshake:456(4315ns) SwitchToState:178(776ns) ProcessPendingSessionEvents:495(4036673ns) {ForwardSessionEventToSession:612(3982397ns) ForwardSessionEventToSession:612(15353ns) ForwardSessionEventToSession:612(2380ns) ForwardSessionEventToSession:612(1864ns) ForwardSessionEventToSession:612(4396ns) ForwardSessionEventToSession:612(1977ns) ForwardSessionEventToSession:612(2549ns) ForwardSessionEventToSession:612(3862ns) ForwardSessionEventToSession:612(2995ns) ForwardSessionEventToSession:612(7566ns)}}}
2025-11-11T17:18:19.769753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518388065398098:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:19.777900Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518388065398098:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:19.797668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518383770430782:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:19.801668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:19.806534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518383770430782:2077] [node 4] ICP22 created new session: [2:7571518388065398116:2077]
2025-11-11T17:18:19.806593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518388065398116:2077] [node 4] ICS09 handshake done sender: [2:7571518388065398098:2117] self: [2:7571518388065398097:0] peer: [4:7571518388145234123:0] socket: 99 qp: -1
2025-11-11T17:18:19.806626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518388065398116:2077] [node 4] ICS10 traffic start
2025-11-11T17:18:19.806673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:19.814037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:19.814080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518383850266730:2049] [node 2] ICP22 created new session: [4:7571518388145234127:2049]
2025-11-11T17:18:19.814118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518388145234127:2049] [node 2] ICS09 handshake done sender: [4:7571518388145234112:2118] self: [4:7571518388145234123:0] peer: [2:7571518388065398097:0] socket: 134 qp: -1
2025-11-11T17:18:19.814147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518388145234127:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:19.814212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:19.928556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:19.928789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518388065398107:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-11-11T17:18:19.928825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518383770430719:2048] [node 1] ICP30 unregister session Session# [2:7571518388065398107:2048] VirtualId# [2:7571518388065398091:0]
2025-11-11T17:18:19.928859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518388065398107:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:19.928937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:19.928976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:19.933717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:19.933946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518385001742769:2048] [node 1] ICS07 socket disconnect 111 reason# EndOfStream
2025-11-11T17:18:19.933971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518385001742769:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:19.934045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:19.934081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518385001742769:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:19.934906Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address
2025-11-11T17:18:19.935148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518380706775380:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:19.935186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518385001742769:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:19.935218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518380706775380:2048] [node 1] ICP30 unregister session Session# [3:7571518385001742769:2048] VirtualId# [3:7571518385001742758:0]
2025-11-11T17:18:19.935257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:19.935280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518380706775380:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:19.935293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:19.941729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:19.941729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:19.941993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518388145234120:2048] [node 1] ICS07 socket disconnect 133 reason# EndOfStream
2025-11-11T17:18:19.942020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518388145234120:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:19.942099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:19.942131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518388145234120:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:19.942790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518386624813207:2048] [node 1] ICS01 socket: 94 reason# EndOfStream
2025-11-11T17:18:19.942830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518382329845821:2048] [node 1] ICP30 unregister session Session# [5:7571518386624813207:2048] VirtualId# [5:7571518386624813198:0]
2025-11-11T17:18:19.942867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518386624813207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:19.942926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:19.942965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:19.943155Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address
2025-11-11T17:18:19.945082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518383850266729:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:19.945133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518388145234120:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:19.945167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518383850266729:2048] [node 1] ICP30 unregister session Session# [4:7571518388145234120:2048] VirtualId# [4:7571518388145234109:0]
2025-11-11T17:18:19.945209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:19.945249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518383850266729:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:19.945266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:19.945733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518380706775380:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:19.945727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:19.945994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518388337809188:2048] [node 1] ICS07 socket disconnect 91 reason# EndOfStream
2025-11-11T17:18:19.946038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518388337809188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:19.946151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:19.946214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518388337809188:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:19.963240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518383850266729:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:19.973819Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address
2025-11-11T17:18:19.974174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518384042841810:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:19.974227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518388337809188:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:19.974257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518384042841810:2048] [node 1] ICP30 unregister session Session# [6:7571518388337809188:2048] VirtualId# [6:7571518388337809184:0]
2025-11-11T17:18:19.974304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:19.974334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518384042841810:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:19.974347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20375) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:19.988292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518384042841810:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:20.005766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:20.005998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518385001742764:2049] [node 2] ICS01 socket: 79 reason# EndOfStream
2025-11-11T17:18:20.006038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518380706775381:2049] [node 2] ICP30 unregister session Session# [3:7571518385001742764:2049] VirtualId# [3:7571518385001742757:0]
2025-11-11T17:18:20.006069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518385001742764:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.006145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:20.006188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:20.029169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:20.029394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518388145234127:2049] [node 2] ICS01 socket: 134 reason# EndOfStream
2025-11-11T17:18:20.029433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518383850266730:2049] [node 2] ICP30 unregister session Session# [4:7571518388145234127:2049] VirtualId# [4:7571518388145234123:0]
2025-11-11T17:18:20.029465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518388145234127:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.029606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:20.029663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:20.029768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:20.030004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518386624813209:2049] [node 2] ICS07 socket disconnect 95 reason# EndOfStream
2025-11-11T17:18:20.030030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518386624813209:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.030126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:20.030180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518386624813209:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:20.030307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:20.030492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518388337809202:2049] [node 2] ICS01 socket: 126 reason# EndOfStream
2025-11-11T17:18:20.030525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518384042841811:2049] [node 2] ICP30 unregister session Session# [6:7571518388337809202:2049] VirtualId# [6:7571518388337809194:0]
2025-11-11T17:18:20.030552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518388337809202:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.030598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:20.030637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:20.031506Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6912) Couldn't connect to any resolved address
2025-11-11T17:18:20.031830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518382329845822:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:20.031917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518386624813209:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:20.031947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518382329845822:2049] [node 2] ICP30 unregister session Session# [5:7571518386624813209:2049] VirtualId# [5:7571518386624813204:0]
2025-11-11T17:18:20.031988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:20.032014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518382329845822:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:20.032027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:20.043015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518382329845822:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:20.099499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:20.099774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518388145234124:2050] [node 3] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-11T17:18:20.099800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518388145234124:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.099899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:20.099940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518388145234124:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:20.100726Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21732) Couldn't connect to any resolved address
2025-11-11T17:18:20.100983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518383850266731:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21732) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:20.101023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518388145234124:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:20.101056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518383850266731:2050] [node 3] ICP30 unregister session Session# [4:7571518388145234124:2050] VirtualId# [4:7571518388145234116:0]
2025-11-11T17:18:20.101096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:20.101119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518383850266731:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:20.101133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:20.109903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518383850266731:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:20.109811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:20.110090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518386624813217:2050] [node 3] ICS01 socket: 115 reason# EndOfStream
2025-11-11T17:18:20.110125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518382329845823:2050] [node 3] ICP30 unregister session Session# [5:7571518386624813217:2050] VirtualId# [5:7571518386624813212:0]
2025-11-11T17:18:20.110153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518386624813217:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.110272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:20.110315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:20.109887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:20.110090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518388337809209:2050] [node 3] ICS01 socket: 102 reason# EndOfStream
2025-11-11T17:18:20.110127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518384042841812:2050] [node 3] ICP30 unregister session Session# [6:7571518388337809209:2050] VirtualId# [6:7571518388337809204:0]
2025-11-11T17:18:20.110153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518388337809209:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.110268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:20.110315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:20.165759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:20.166051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518388337809207:2059] [node 4] ICS01 socket: 113 reason# EndOfStream
2025-11-11T17:18:20.166126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518384042841827:2059] [node 4] ICP30 unregister session Session# [6:7571518388337809207:2059] VirtualId# [6:7571518388337809193:0]
2025-11-11T17:18:20.166168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518388337809207:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.166277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:20.166338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:20.165753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:20.166114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518386624813213:2061] [node 4] ICS01 socket: 83 reason# EndOfStream
2025-11-11T17:18:20.166184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518386624813179:2061] [node 4] ICP30 unregister session Session# [5:7571518386624813213:2061] VirtualId# [5:7571518386624813200:0]
2025-11-11T17:18:20.166254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518386624813213:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:20.166437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:20.166527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:20.449883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518392822961229:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:20.449925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518392822961230:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:20.553889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518392043014176:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:20.553941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518392043014177:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:20.604363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518390434346764:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:20.604407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518390434346765:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:20.665868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518391871504428:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:20.665902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518391871504429:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:20.665924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518391871504430:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:20.761912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518392826371292:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:20.761948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518392826371293:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:20.761971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518392826371294:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:20.841896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518392938187469:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:20.841943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518392938187470:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:20.841970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518392938187471:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:20.962533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518392938187482:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.017101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518396337981486:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47385
2025-11-11T17:18:21.029046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518396337981489:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:20.971344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518392938187488:2062] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.115816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338605:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.102354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518396166471740:2060] [node 5] ICP01 ready to work
2025-11-11T17:18:21.083377Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518392822961230:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(53464919ns) {Configure:104(53461812ns) {ProcessConfigured:135(68009ns) {StartInitialHandshake:157(65686ns) {DropHandshakes:469(4002ns) {DropIncomingHandshake:441(781ns) DropOutgoingHandshake:456(735ns)} GenerateSessionVirtualId:988(768ns) PrepareNewSessionHandshake:476(2084ns) {SwitchToState:178(719ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.221347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518394729314087:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14091
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.226868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154797:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.130644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518396337981503:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.083818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518397117928548:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4975
2025-11-11T17:18:21.345974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518397121338623:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:21.346395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518397121338643:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:21.201938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518394729314085:2060] [node 5] ICP01 ready to work
2025-11-11T17:18:21.262269Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518397233154797:2073] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.341966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154828:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.210207Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981486:2062] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.233691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518396166471753:2066] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.153786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518397117928558:2063] [node 6] ICP01 ready to work
2025-11-11T17:18:21.325777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518394729314091:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.210300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518392043014176:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.310547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471748:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.462254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154841:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.327401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518394729314092:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14091
2025-11-11T17:18:21.209978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928548:2068] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.394526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518397121338623:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(47713441ns) {Configure:104(47708680ns) {ProcessConfigured:135(54310ns) {StartInitialHandshake:157(51767ns) {DropHandshakes:469(3348ns) {DropIncomingHandshake:441(835ns) DropOutgoingHandshake:456(485ns)} GenerateSessionVirtualId:988(756ns) PrepareNewSessionHandshake:476(2246ns) {SwitchToState:178(698ns)}}}}}
2025-11-11T17:18:21.211419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518396337981512:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47385
2025-11-11T17:18:21.395680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518397121338605:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.334182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471747:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
2025-11-11T17:18:21.212029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518396337981486:2062] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:21.212977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518392043014176:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.213007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:21.213040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518392043014176:2048] [node 1] ICP22 created new session: [2:7571518396337981513:2048]
2025-11-11T17:18:21.213074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518396337981513:2048] [node 1] ICS09 handshake done sender: [2:7571518396337981486:2062] self: [2:7571518396337981511:0] peer: [1:7571518397117928538:0] socket: 87 qp: -1
2025-11-11T17:18:21.213096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518396337981513:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:21.213119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:21.189657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518396337981507:2060] [node 5] ICP01 ready to work
2025-11-11T17:18:21.411705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518396337981550:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47385
2025-11-11T17:18:21.499089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518396337981553:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47385
2025-11-11T17:18:21.500938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981553:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.574419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518396337981489:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(545038541ns) {Configure:104(545033271ns) {ProcessConfigured:135(65983ns) {StartInitialHandshake:157(63123ns) {DropHandshakes:469(3070ns) {DropIncomingHandshake:441(784ns) DropOutgoingHandshake:456(494ns)} GenerateSessionVirtualId:988(1044ns) PrepareNewSessionHandshake:476(2616ns) {SwitchToState:178(1128ns)}}}}}
2025-11-11T17:18:21.577722Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981550:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.483942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518397233154841:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.210501Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928539:2061] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.210521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518397117928539:2061] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.413698Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314087:2073] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.461081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338661:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.633703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981505:2075] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.633754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518396337981505:2075] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.636179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981549:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.636199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518396337981549:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.636727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518396337981557:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47385
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.649943Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518396337981549:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:21.484071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518392938187488:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.211562Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518397117928539:2061] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:21.446433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518390434346765:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(484469836ns) {Configure:104(480955631ns) {ProcessConfigured:135(67834ns) {StartInitialHandshake:157(60278ns) {DropHandshakes:469(3107ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(658ns)} GenerateSessionVirtualId:988(1156ns) PrepareNewSessionHandshake:476(2478ns) {SwitchToState:178(821ns)}}}}}
2025-11-11T17:18:21.412359Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518396166471747:2069] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.461126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338662:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.508761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154843:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.674559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518396337981507:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.674669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:21.674716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518396337981507:2060] [node 5] ICP22 created new session: [2:7571518396337981562:2060]
2025-11-11T17:18:21.674780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518396337981562:2060] [node 5] ICS09 handshake done sender: [2:7571518396337981549:2113] self: [2:7571518396337981548:0] peer: [5:7571518397121338673:0] socket: 110 qp: -1
2025-11-11T17:18:21.674815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518396337981562:2060] [node 5] ICS10 traffic start
2025-11-11T17:18:21.674854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:21.675878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518392043014177:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(681400088ns) {Configure:104(681394854ns) {ProcessConfigured:135(21390301ns) {StartInitialHandshake:157(21387733ns) {DropHandshakes:469(3155ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(941ns) PrepareNewSessionHandshake:476(2799ns) {SwitchToState:178(443ns)}}}}}
2025-11-11T17:18:21.677873Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981557:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.212259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518392822961229:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.461500Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518397121338662:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.447024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518394729314102:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14091
2025-11-11T17:18:21.539408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518396166471740:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(417401191ns) {Configure:104(417396201ns) {ProcessConfigured:135(59426ns) {StartInitialHandshake:157(57609ns) {DropHandshakes:469(3172ns) {DropIncomingHandshake:441(912ns) DropOutgoingHandshake:456(622ns)} GenerateSessionVirtualId:988(343ns) PrepareNewSessionHandshake:476(5190ns) {SwitchToState:178(2765ns)}}}}}
2025-11-11T17:18:21.212306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:21.509400Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518397233154841:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:21.693761Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518396337981505:2075] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.447099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518390434346764:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.506281Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518397121338657:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.540177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471800:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
2025-11-11T17:18:21.541997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518396166471748:2070] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.549751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518396166471800:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.550801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471804:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
2025-11-11T17:18:21.589634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518391871504429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.602321Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518396166471757:2075] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.602342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518396166471757:2075] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.602953Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518396166471804:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.603152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518391871504430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.603382Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518396166471757:2075] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:21.603596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518396166471753:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.603637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:21.603670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518396166471753:2066] [node 6] ICP22 created new session: [4:7571518396166471809:2066]
2025-11-11T17:18:21.603707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518396166471809:2066] [node 6] ICS09 handshake done sender: [4:7571518396166471757:2075] self: [4:7571518396166471756:0] peer: [6:7571518397233154847:0] socket: 103 qp: -1
2025-11-11T17:18:21.603736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518396166471809:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:21.603763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:21.647124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518391871504429:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31923265ns) {IncomingHandshake:261(31920998ns) {DropIncomingHandshake:441(474ns) PrepareNewSessionHandshake:476(2018ns) {SwitchToState:178(907ns)} GenerateSessionVirtualId:988(441ns) IssueIncomingHandshakeReply:193(20013019ns) {SwitchToState:178(2541ns)}}}
2025-11-11T17:18:21.650226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518391871504428:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(652234106ns) {Configure:104(652230075ns) {ProcessConfigured:135(12978898ns) {StartInitialHandshake:157(12975088ns) {DropHandshakes:469(3144ns) {DropIncomingHandshake:441(895ns) DropOutgoingHandshake:456(481ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:476(2879ns) {SwitchToState:178(1049ns)}}}}}
2025-11-11T17:18:21.661920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471814:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
2025-11-11T17:18:21.687422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518396166471802:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.687444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518396166471802:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.688400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471815:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
2025-11-11T17:18:21.690832Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518396166471802:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:21.691057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518396166471740:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.691112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:21.691153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518396166471740:2060] [node 5] ICP22 created new session: [4:7571518396166471817:2060]
2025-11-11T17:18:21.691195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518396166471817:2060] [node 5] ICS09 handshake done sender: [4:7571518396166471802:2116] self: [4:7571518396166471801:0] peer: [5:7571518397121338682:0] socket: 92 qp: -1
2025-11-11T17:18:21.691225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518396166471817:2060] [node 5] ICS10 traffic start
2025-11-11T17:18:21.691246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:21.715874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518396166471804:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:21.721654Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518396166471748:2070] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:21.212341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518392822961229:2048] [node 2] ICP22 created new session: [1:7571518397117928569:2048]
2025-11-11T17:18:21.509523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518392938187488:2062] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.702231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518396337981507:2060] [node 5] ICP18 handshake (actor [2:7571518396337981557:2118]) is too old
2025-11-11T17:18:21.506306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518397121338657:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.453933Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.708546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [2:7571518396337981507:2060] [node 5] ICP35 event processing took too much time StateWork:293(16479636ns) {IncomingHandshake:261(16477282ns)}
2025-11-11T17:18:21.212379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518397117928569:2048] [node 2] ICS09 handshake done sender: [1:7571518397117928539:2061] self: [1:7571518397117928538:0] peer: [2:7571518396337981511:0] socket: 86 qp: -1
2025-11-11T17:18:21.735178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518391871504430:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.509565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:21.508954Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518397121338657:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.509652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518392938187488:2062] [node 5] ICP22 created new session: [6:7571518397233154844:2062]
2025-11-11T17:18:21.212413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518397117928569:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:21.716400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518396337981503:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.454831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518390434346764:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518394729314087:2073] held: no
2025-11-11T17:18:21.737633Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.510130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518392826371294:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(504150019ns) {Configure:104(504145522ns) {ProcessConfigured:135(31851268ns) {StartInitialHandshake:157(31847809ns) {DropHandshakes:469(3066ns) {DropIncomingHandshake:441(731ns) DropOutgoingHandshake:456(678ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:476(3320ns) {SwitchToState:178(1270ns)}}}}}
2025-11-11T17:18:21.716448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.212441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:21.509704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518397233154844:2062] [node 5] ICS09 handshake done sender: [6:7571518397233154841:2115] self: [6:7571518397233154842:0] peer: [5:7571518397121338656:0] socket: 114 qp: -1
2025-11-11T17:18:21.740130Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6585) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.753825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518391871504429:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.753872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:21.753912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518391871504429:2049] [node 2] ICP22 created new session: [4:7571518396166471820:2049]
2025-11-11T17:18:21.753950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518396166471820:2049] [node 2] ICS09 handshake done sender: [4:7571518396166471748:2070] self: [4:7571518396166471805:0] peer: [2:7571518396337981504:0] socket: 95 qp: -1
2025-11-11T17:18:21.753985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518396166471820:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:21.754018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:21.754240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518391871504429:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:21.762700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:21.767315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518391871504430:2050] [node 3] ICP22 created new session: [4:7571518396166471822:2050]
2025-11-11T17:18:21.782011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518396166471822:2050] [node 3] ICS09 handshake done sender: [4:7571518396166471804:2117] self: [4:7571518396166471808:0] peer: [3:7571518394729314093:0] socket: 122 qp: -1
2025-11-11T17:18:21.785126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518396166471822:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:21.786056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:21.786492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518391871504430:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(52008304ns) {HandleHandshakeStatus:329(51084935ns) {DropIncomingHandshake:441(1347ns) DropOutgoingHandshake:456(5326ns) SwitchToState:178(1030ns) ProcessPendingSessionEvents:495(98170ns) {ForwardSessionEventToSession:612(45323ns) ForwardSessionEventToSession:612(9196ns) ForwardSessionEventToSession:612(2359ns) ForwardSessionEventToSession:612(2673ns) ForwardSessionEventToSession:612(4133ns) ForwardSessionEventToSession:612(2716ns) ForwardSessionEventToSession:612(2294ns) ForwardSessionEventToSession:612(12606ns) ForwardSessionEventToSession:612(2106ns) ForwardSessionEventToSession:612(4375ns)}}}
2025-11-11T17:18:21.786614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518391871504430:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:21.510332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518392826371294:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.455020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518394729314115:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14091
2025-11-11T17:18:21.226459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518397117928572:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4975
2025-11-11T17:18:21.716489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518396337981503:2068] [node 4] ICP22 created new session: [2:7571518396337981567:2068]
2025-11-11T17:18:21.455458Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518394729314087:2073] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:21.547920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518397121338643:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.509749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518397233154844:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:21.537676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:21.545653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518397233154828:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.594103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518392938187488:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(84384357ns) {HandleHandshakeStatus:329(84382732ns) {DropIncomingHandshake:441(905ns) DropOutgoingHandshake:456(6320ns) SwitchToState:178(608ns) ProcessPendingSessionEvents:495(118627ns) {ForwardSessionEventToSession:612(65879ns) ForwardSessionEventToSession:612(6439ns) ForwardSessionEventToSession:612(3027ns) ForwardSessionEventToSession:612(6194ns) ForwardSessionEventToSession:612(1746ns) ForwardSessionEventToSession:612(6127ns) ForwardSessionEventToSession:612(6691ns) ForwardSessionEventToSession:612(2182ns) ForwardSessionEventToSession:612(2098ns) ForwardSessionEventToSession:612(9597ns)}}}
2025-11-11T17:18:21.594786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518392938187482:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.594984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154846:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.610552Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62534) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.455624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518390434346764:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.455685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:21.455719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518390434346764:2048] [node 1] ICP22 created new session: [3:7571518394729314117:2048]
2025-11-11T17:18:21.455750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518394729314117:2048] [node 1] ICS09 handshake done sender: [3:7571518394729314087:2073] self: [3:7571518394729314112:0] peer: [1:7571518397117928546:0] socket: 93 qp: -1
2025-11-11T17:18:21.455778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518394729314117:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:21.455807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:21.489020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314092:2075] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.489127Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314102:2083] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.505732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518394729314085:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(66510238ns) {Configure:104(66499785ns) {ProcessConfigured:135(64669729ns) {StartInitialHandshake:157(64667493ns) {DropHandshakes:469(23280041ns) {DropIncomingHandshake:441(1377ns) DropOutgoingHandshake:456(224ns)} GenerateSessionVirtualId:988(1263ns) PrepareNewSessionHandshake:476(2337ns) {SwitchToState:178(1201ns)}}}}}
2025-11-11T17:18:21.546029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518394729314143:2079] [node 6] ICP01 ready to work
2025-11-11T17:18:21.553999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518394729314147:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14091
2025-11-11T17:18:21.565941Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314114:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.565979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518394729314114:2091] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.597757Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518394729314114:2091] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:21.598231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518394729314085:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.598301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:21.598347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518394729314085:2060] [node 5] ICP22 created new session: [3:7571518394729314151:2060]
2025-11-11T17:18:21.598388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518394729314151:2060] [node 5] ICS09 handshake done sender: [3:7571518394729314114:2091] self: [3:7571518394729314105:0] peer: [5:7571518397121338666:0] socket: 89 qp: -1
2025-11-11T17:18:21.598417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518394729314151:2060] [node 5] ICS10 traffic start
2025-11-11T17:18:21.598455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:21.618536Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314147:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.618643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518394729314085:2060] [node 5] ICP18 handshake (actor [3:7571518394729314147:2117]) is too old
2025-11-11T17:18:21.637037Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314094:2076] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.637068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518394729314094:2076] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.663088Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518394729314094:2076] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:21.711498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518394729314091:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.711546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:21.711586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518394729314091:2061] [node 4] ICP22 created new session: [3:7571518394729314156:2061]
2025-11-11T17:18:21.711628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518394729314156:2061] [node 4] ICS09 handshake done sender: [3:7571518394729314094:2076] self: [3:7571518394729314093:0] peer: [4:7571518396166471808:0] socket: 111 qp: -1
2025-11-11T17:18:21.711657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518394729314156:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:21.711686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:21.713728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314145:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.713760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518394729314145:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.719883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518394729314155:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14091
2025-11-11T17:18:21.733833Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518394729314145:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:21.767441Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518394729314155:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.777837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518390434346765:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.782098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518394729314143:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.790062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:21.804232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518394729314143:2079] [node 6] ICP22 created new session: [3:7571518394729314160:2079]
2025-11-11T17:18:21.806057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518394729314161:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14091
2025-11-11T17:18:21.829985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518394729314160:2079] [node 6] ICS09 handshake done sender: [3:7571518394729314145:2116] self: [3:7571518394729314144:0] peer: [6:7571518397233154852:0] socket: 128 qp: -1
2025-11-11T17:18:21.833963Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518394729314155:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:21.835372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518390434346765:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.835424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:21.835461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518390434346765:2049] [node 2] ICP22 created new session: [3:7571518394729314162:2049]
2025-11-11T17:18:21.835499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518394729314162:2049] [node 2] ICS09 handshake done sender: [3:7571518394729314155:2120] self: [3:7571518394729314159:0] peer: [2:7571518396337981560:0] socket: 118 qp: -1
2025-11-11T17:18:21.835521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518394729314162:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:21.835557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:21.841205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518394729314160:2079] [node 6] ICS10 traffic start
2025-11-11T17:18:21.850649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:21.261664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518397117928560:2058] [node 5] ICP01 ready to work
2025-11-11T17:18:21.716527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518396337981567:2068] [node 4] ICS09 handshake done sender: [2:7571518396337981505:2075] self: [2:7571518396337981504:0] peer: [4:7571518396166471805:0] socket: 94 qp: -1
2025-11-11T17:18:21.611475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518392938187482:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62534) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518397233154828:2103] held: no
2025-11-11T17:18:21.611649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154848:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.611765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154849:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.612158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518397233154828:2103] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:21.612305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518392938187482:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.612355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:21.612396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518392938187482:2059] [node 4] ICP22 created new session: [6:7571518397233154850:2059]
2025-11-11T17:18:21.612438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518397233154850:2059] [node 4] ICS09 handshake done sender: [6:7571518397233154828:2103] self: [6:7571518397233154847:0] peer: [4:7571518396166471756:0] socket: 105 qp: -1
2025-11-11T17:18:21.612469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518397233154850:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:21.612498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:21.650484Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518397233154848:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.678063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518392938187471:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.686573Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518397233154846:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.718120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518392938187471:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(23918139ns) {IncomingHandshake:261(14934168ns) {DropIncomingHandshake:441(1236ns) PrepareNewSessionHandshake:476(3119ns) {SwitchToState:178(2626ns)} GenerateSessionVirtualId:988(662ns) IssueIncomingHandshakeReply:193(40845ns) {SwitchToState:178(331ns)}}}
2025-11-11T17:18:21.722369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518392938187469:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(488283930ns) {Configure:104(488279372ns) {ProcessConfigured:135(47739ns) {StartInitialHandshake:157(45680ns) {DropHandshakes:469(1356ns) {DropIncomingHandshake:441(352ns) DropOutgoingHandshake:456(194ns)} GenerateSessionVirtualId:988(574ns) PrepareNewSessionHandshake:476(1252ns) {SwitchToState:178(268ns)}}}}}
2025-11-11T17:18:21.722617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518392938187469:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.722918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154855:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.742132Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518397233154848:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:21.742339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518392938187471:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.742408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:21.742451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518392938187471:2050] [node 3] ICP22 created new session: [6:7571518397233154858:2050]
2025-11-11T17:18:21.742488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518397233154858:2050] [node 3] ICS09 handshake done sender: [6:7571518397233154848:2118] self: [6:7571518397233154852:0] peer: [3:7571518394729314144:0] socket: 131 qp: -1
2025-11-11T17:18:21.742521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518397233154858:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:21.742555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:21.749915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518392938187470:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(243660916ns) {Configure:104(243650392ns) {ProcessConfigured:135(62440ns) {StartInitialHandshake:157(59462ns) {DropHandshakes:469(2817ns) {DropIncomingHandshake:441(656ns) DropOutgoingHandshake:456(693ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:476(1705ns) {SwitchToState:178(471ns)}}}}}
2025-11-11T17:18:21.750152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518392938187470:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.779773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154864:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.789760Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518397233154846:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:21.813958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518392938187470:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.814037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:21.814089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518392938187470:2049] [node 2] ICP22 created new session: [6:7571518397233154866:2049]
2025-11-11T17:18:21.814138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518397233154866:2049] [node 2] ICS09 handshake done sender: [6:7571518397233154846:2117] self: [6:7571518397233154862:0] peer: [2:7571518396337981554:0] socket: 127 qp: -1
2025-11-11T17:18:21.814191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518397233154866:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:21.814231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:21.849659Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.850384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518392938187469:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518397233154797:2073] held: no
2025-11-11T17:18:21.547975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:21.716556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518396337981567:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:21.716581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:21.750877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981555:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.750898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518396337981555:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.751725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518396337981569:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47385
2025-11-11T17:18:21.751952Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518396337981555:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:21.752159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518396337981489:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.752206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:21.752256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518396337981489:2064] [node 6] ICP22 created new session: [2:7571518396337981571:2064]
2025-11-11T17:18:21.752294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518396337981571:2064] [node 6] ICS09 handshake done sender: [2:7571518396337981555:2117] self: [2:7571518396337981554:0] peer: [6:7571518397233154862:0] socket: 125 qp: -1
2025-11-11T17:18:21.752322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518396337981571:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:21.752357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:21.784693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981564:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.784722Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518396337981564:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.802030Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518396337981564:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:21.834288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518392043014177:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.834368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:21.834413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518392043014177:2049] [node 3] ICP22 created new session: [2:7571518396337981574:2049]
2025-11-11T17:18:21.834455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518396337981574:2049] [node 3] ICS09 handshake done sender: [2:7571518396337981564:2119] self: [2:7571518396337981560:0] peer: [3:7571518394729314159:0] socket: 138 qp: -1
2025-11-11T17:18:21.834485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518396337981574:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:21.834513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:21.855365Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518396337981569:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.548008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518397121338643:2069] [node 6] ICP22 created new session: [5:7571518397121338669:2069]
2025-11-11T17:18:21.548054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518397121338669:2069] [node 6] ICS09 handshake done sender: [5:7571518397121338657:2112] self: [5:7571518397121338656:0] peer: [6:7571518397233154842:0] socket: 109 qp: -1
2025-11-11T17:18:21.548083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518397121338669:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:21.548114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:21.548897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518392826371293:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(542884004ns) {Configure:104(542880790ns) {ProcessConfigured:135(14568774ns) {StartInitialHandshake:157(14566651ns) {DropHandshakes:469(3127ns) {DropIncomingHandshake:441(576ns) DropOutgoingHandshake:456(618ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:476(2152ns) {SwitchToState:178(105ns)}}}}}
2025-11-11T17:18:21.577929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518397121338661:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.602169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338674:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.605779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518392826371293:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.618375Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518397121338662:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:21.621873Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.621906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518392826371294:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.621972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:21.622032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518392826371294:2050] [node 3] ICP22 created new session: [5:7571518397121338676:2050]
2025-11-11T17:18:21.622076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518397121338676:2050] [node 3] ICS09 handshake done sender: [5:7571518397121338662:2114] self: [5:7571518397121338666:0] peer: [3:7571518394729314105:0] socket: 116 qp: -1
2025-11-11T17:18:21.622895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338675:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.630239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518397121338676:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:21.637355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:21.638241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518392826371293:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19064688ns) {IncomingHandshake:261(19062632ns) {DropIncomingHandshake:441(650ns) PrepareNewSessionHandshake:476(2765ns) {SwitchToState:178(1136ns)} GenerateSessionVirtualId:988(572ns) IssueIncomingHandshakeReply:193(37056ns) {SwitchToState:178(255ns)}}}
2025-11-11T17:18:21.642103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338678:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.654339Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518397121338675:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.657933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518392826371292:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(439478577ns) {Configure:104(439473410ns) {ProcessConfigured:135(77478ns) {StartInitialHandshake:157(75855ns) {DropHandshakes:469(1501ns) {DropIncomingHandshake:441(333ns) DropOutgoingHandshake:456(195ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:476(1994ns) {SwitchToState:178(552ns)}}}}}
2025-11-11T17:18:21.676629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518392826371294:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(54539324ns) {HandleHandshakeStatus:329(54536721ns) {DropIncomingHandshake:441(1034ns) DropOutgoingHandshake:456(3618ns) SwitchToState:178(612ns) ProcessPendingSessionEvents:495(22113386ns) {ForwardSessionEventToSession:612(22063834ns) ForwardSessionEventToSession:612(8330ns) ForwardSessionEventToSession:612(4733ns) ForwardSessionEventToSession:612(2919ns) ForwardSessionEventToSession:612(2486ns) ForwardSessionEventToSession:612(7261ns) ForwardSessionEventToSession:612(2111ns) ForwardSessionEventToSession:612(4349ns) ForwardSessionEventToSession:612(2823ns) ForwardSessionEventToSession:612(3998ns)}}}
2025-11-11T17:18:21.676689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518392826371294:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:21.686036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518397121338623:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.687158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518397121338661:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:21.688866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338683:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.689168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518392826371293:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.689238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:21.689281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518392826371293:2049] [node 2] ICP22 created new session: [5:7571518397121338684:2049]
2025-11-11T17:18:21.689319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518397121338684:2049] [node 2] ICS09 handshake done sender: [5:7571518397121338661:2113] self: [5:7571518397121338673:0] peer: [2:7571518396337981548:0] socket: 115 qp: -1
2025-11-11T17:18:21.689349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518397121338684:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:21.689401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:21.714091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518397121338623:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(22761206ns) {IncomingHandshake:261(7941237ns) {DropIncomingHandshake:441(419ns) PrepareNewSessionHandshake:476(520ns) {SwitchToState:178(120ns)} GenerateSessionVirtualId:988(335ns) IssueIncomingHandshakeReply:193(53835ns) {SwitchToState:178(1086ns)}}}
2025-11-11T17:18:21.714279Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518397121338675:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:21.718494Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62534) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.720177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518397121338623:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62534) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518397121338675:2118] held: no
2025-11-11T17:18:21.720406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338686:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.736175Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518397121338686:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.736226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518397121338623:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.741708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:21.758281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518392826371292:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.765776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518397121338623:2059] [node 4] ICP22 created new session: [5:7571518397121338687:2059]
2025-11-11T17:18:21.769754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518397121338687:2059] [node 4] ICS09 handshake done sender: [5:7571518397121338675:2118] self: [5:7571518397121338682:0] peer: [4:7571518396166471801:0] socket: 84 qp: -1
2025-11-11T17:18:21.769791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518397121338687:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:21.769840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:21.770031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518397121338623:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(33722507ns) {HandleHandshakeStatus:329(33720309ns) {DropIncomingHandshake:441(310ns) DropOutgoingHandshake:456(171ns) SwitchToState:178(2073ns) ProcessPendingSessionEvents:495(981ns)}}
2025-11-11T17:18:21.783209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518397121338690:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-11T17:18:21.793696Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518397121338686:2122] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:21.798483Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:21.799085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518392826371292:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.799151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:21.799195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518392826371292:2048] [node 1] ICP22 created new session: [5:7571518397121338691:2048]
2025-11-11T17:18:21.799240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518397121338691:2048] [node 1] ICS09 handshake done sender: [5:7571518397121338686:2122] self: [5:7571518397121338688:0] peer: [1:7571518397117928612:0] socket: 142 qp: -1
2025-11-11T17:18:21.799269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518397121338691:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:21.799297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:21.799523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518392826371292:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:21.345628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518397117928566:2072] [node 4] ICP01 ready to work
2025-11-11T17:18:21.449801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928572:2082] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.449893Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928547:2067] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.449910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518397117928547:2067] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.450701Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518397117928547:2067] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:21.450881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518392822961230:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.450930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:21.450971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518392822961230:2049] [node 3] ICP22 created new session: [1:7571518397117928607:2049]
2025-11-11T17:18:21.451004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518397117928607:2049] [node 3] ICS09 handshake done sender: [1:7571518397117928547:2067] self: [1:7571518397117928546:0] peer: [3:7571518394729314112:0] socket: 90 qp: -1
2025-11-11T17:18:21.451030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518397117928607:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:21.451058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:21.662381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518397117928610:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4975
2025-11-11T17:18:21.697850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518397117928611:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4975
2025-11-11T17:18:21.710993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518397117928560:2058] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(321119871ns) {Configure:104(321112967ns) {ProcessConfigured:135(62802ns) {StartInitialHandshake:157(58776ns) {DropHandshakes:469(3268ns) {DropIncomingHandshake:441(870ns) DropOutgoingHandshake:456(552ns)} GenerateSessionVirtualId:988(636ns) PrepareNewSessionHandshake:476(2186ns) {SwitchToState:178(1021ns)}}}}}
2025-11-11T17:18:21.714409Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928610:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.721889Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928611:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.734909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928563:2079] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.734939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518397117928563:2079] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.738667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518397117928616:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4975
2025-11-11T17:18:21.765912Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928613:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.765935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518397117928613:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.767020Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518397117928613:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:21.784335Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928616:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.785441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518397117928566:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(395509472ns) {Configure:104(371627285ns) {ProcessConfigured:135(46091ns) {StartInitialHandshake:157(43658ns) {DropHandshakes:469(4085ns) {DropIncomingHandshake:441(374ns) DropOutgoingHandshake:456(264ns)} GenerateSessionVirtualId:988(408ns) PrepareNewSessionHandshake:476(994ns) {SwitchToState:178(354ns)}}}}}
2025-11-11T17:18:21.788995Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518397117928563:2079] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:21.793880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518397117928560:2058] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.793941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:21.793985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518397117928560:2058] [node 5] ICP22 created new session: [1:7571518397117928623:2058]
2025-11-11T17:18:21.794062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518397117928623:2058] [node 5] ICS09 handshake done sender: [1:7571518397117928613:2117] self: [1:7571518397117928612:0] peer: [5:7571518397121338688:0] socket: 91 qp: -1
2025-11-11T17:18:21.794094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518397117928623:2058] [node 5] ICS10 traffic start
2025-11-11T17:18:21.794128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:21.798347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518397117928558:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.798417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:21.798454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518397117928558:2063] [node 6] ICP22 created new session: [1:7571518397117928625:2063]
2025-11-11T17:18:21.798491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518397117928625:2063] [node 6] ICS09 handshake done sender: [1:7571518397117928563:2079] self: [1:7571518397117928562:0] peer: [6:7571518397233154856:0] socket: 99 qp: -1
2025-11-11T17:18:21.798521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518397117928625:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:21.798554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:21.818653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518397117928558:2063] [node 6] ICP35 event processing took too much time PendingConnection:278(20130917ns) {HandleHandshakeStatus:329(20128418ns) {DropIncomingHandshake:441(10599ns) DropOutgoingHandshake:456(329ns) SwitchToState:178(1048ns) ProcessPendingSessionEvents:495(19763116ns) {ForwardSessionEventToSession:612(19719832ns) ForwardSessionEventToSession:612(4687ns) ForwardSessionEventToSession:612(2248ns) ForwardSessionEventToSession:612(5547ns) ForwardSessionEventToSession:612(2422ns) ForwardSessionEventToSession:612(5050ns) ForwardSessionEventToSession:612(4479ns) ForwardSessionEventToSession:612(2469ns) ForwardSessionEventToSession:612(4846ns) ForwardSessionEventToSession:612(1801ns)}}}
2025-11-11T17:18:21.871714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518394729314143:2079] [node 6] ICP35 event processing took too much time PendingConnection:278(100339608ns) {HandleHandshakeStatus:329(100336509ns) {DropIncomingHandshake:441(24463ns) DropOutgoingHandshake:456(547ns) SwitchToState:178(756ns) ProcessPendingSessionEvents:495(1638222ns) {ForwardSessionEventToSession:612(1596788ns) ForwardSessionEventToSession:612(6214ns) ForwardSessionEventToSession:612(2266ns) ForwardSessionEventToSession:612(4407ns) ForwardSessionEventToSession:612(2166ns) ForwardSessionEventToSession:612(2148ns) ForwardSessionEventToSession:612(4651ns) ForwardSessionEventToSession:612(1535ns) ForwardSessionEventToSession:612(5027ns) ForwardSessionEventToSession:612(4358ns)}}}
2025-11-11T17:18:21.870145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518396337981489:2064] [node 6] ICP18 handshake (actor [2:7571518396337981569:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.872885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471824:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.882120Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518396166471824:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.882313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518391871504428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:21.883915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518396166471826:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18164
2025-11-11T17:18:21.884663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518396166471824:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:21.884937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518391871504428:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.885024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:21.885069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518391871504428:2048] [node 1] ICP22 created new session: [4:7571518396166471827:2048]
2025-11-11T17:18:21.885106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518396166471827:2048] [node 1] ICS09 handshake done sender: [4:7571518396166471824:2121] self: [4:7571518396166471825:0] peer: [1:7571518397117928620:0] socket: 112 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:21.887042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518397233154865:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53054
2025-11-11T17:18:21.887621Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518397233154797:2073] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:21.887909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518392938187469:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:21.887967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:21.888005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518392938187469:2048] [node 1] ICP22 created new session: [6:7571518397233154868:2048]
2025-11-11T17:18:21.888041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518397233154868:2048] [node 1] ICS09 handshake done sender: [6:7571518397233154797:2073] self: [6:7571518397233154856:0] peer: [1:7571518397117928562:0] socket: 100 qp: -1
2025-11-11T17:18:21.888072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518397233154868:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:21.888108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:21.882684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518397117928621:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:21.882718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518397117928621:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:21.884161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518397117928621:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:21.885139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518396166471827:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:21.902633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518397117928566:2072] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:21.902755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:21.902822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518397117928566:2072] [node 4] ICP22 created new session: [1:7571518397117928629:2072]
2025-11-11T17:18:21.902871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518397117928629:2072] [node 4] ICS09 handshake done sender: [1:7571518397117928621:2119] self: [1:7571518397117928620:0] peer: [4:7571518396166471825:0] socket: 88 qp: -1
2025-11-11T17:18:21.902915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518397117928629:2072] [node 4] ICS10 traffic start
2025-11-11T17:18:21.902971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:21.885172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:21.963118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:21.963360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518396337981513:2048] [node 1] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-11T17:18:21.963392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518396337981513:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:21.963463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:21.963497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518396337981513:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:21.970912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:21.971156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518394729314117:2048] [node 1] ICS07 socket disconnect 93 reason# EndOfStream
2025-11-11T17:18:21.971183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518394729314117:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:21.971238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:21.971270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518394729314117:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:21.973772Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:18:21.974277Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:18:21.974904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518392043014176:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:18:21.974939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518396337981513:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:21.975529Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address
2025-11-11T17:18:21.975710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518392043014176:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:21.975764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518396337981513:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:21.975790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518392043014176:2048] [node 1] ICP30 unregister session Session# [2:7571518396337981513:2048] VirtualId# [2:7571518396337981511:0]
2025-11-11T17:18:21.975844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:21.975873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518392043014176:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:21.975886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:21.981744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:21.981976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518397233154868:2048] [node 1] ICS01 socket: 100 reason# EndOfStream
2025-11-11T17:18:21.982043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518392938187469:2048] [node 1] ICP30 unregister session Session# [6:7571518397233154868:2048] VirtualId# [6:7571518397233154856:0]
2025-11-11T17:18:21.982073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518397233154868:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:21.982151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:21.982191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:21.977724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:21.997725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518392043014176:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:21.985700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:21.977869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518390434346764:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:18:21.984166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518397121338691:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-11T17:18:22.013679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518396166471827:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-11-11T17:18:22.013737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518391871504428:2048] [node 1] ICP30 unregister session Session# [4:7571518396166471827:2048] VirtualId# [4:7571518396166471825:0]
2025-11-11T17:18:22.013773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518396166471827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.013881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:22.013930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:21.977901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518394729314117:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:22.018004Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address
2025-11-11T17:18:22.019567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518390434346764:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:22.019613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518394729314117:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:22.019644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518390434346764:2048] [node 1] ICP30 unregister session Session# [3:7571518394729314117:2048] VirtualId# [3:7571518394729314112:0]
2025-11-11T17:18:22.019689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:22.019715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518390434346764:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:22.019727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:21.984209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518392826371292:2048] [node 1] ICP30 unregister session Session# [5:7571518397121338691:2048] VirtualId# [5:7571518397121338688:0]
2025-11-11T17:18:21.984241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518397121338691:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:21.984320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:21.984359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:22.149742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:22.149997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518394729314162:2049] [node 2] ICS01 socket: 118 reason# EndOfStream
2025-11-11T17:18:22.150046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518390434346765:2049] [node 2] ICP30 unregister session Session# [3:7571518394729314162:2049] VirtualId# [3:7571518394729314159:0]
2025-11-11T17:18:22.150073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518394729314162:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.150154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:22.150193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:22.158240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:22.157779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:22.158016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518397121338684:2049] [node 2] ICS01 socket: 115 reason# EndOfStream
2025-11-11T17:18:22.158047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518392826371293:2049] [node 2] ICP30 unregister session Session# [5:7571518397121338684:2049] VirtualId# [5:7571518397121338673:0]
2025-11-11T17:18:22.158075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518397121338684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.158147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:22.158186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:22.157784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:22.158011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518396166471820:2049] [node 2] ICS07 socket disconnect 95 reason# EndOfStream
2025-11-11T17:18:22.158039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518396166471820:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.158127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:22.158172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518396166471820:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:22.159440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518397233154866:2049] [node 2] ICS01 socket: 127 reason# EndOfStream
2025-11-11T17:18:22.159476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518392938187470:2049] [node 2] ICP30 unregister session Session# [6:7571518397233154866:2049] VirtualId# [6:7571518397233154862:0]
2025-11-11T17:18:22.159506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518397233154866:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.159560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:22.159593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:22.160122Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6585) Couldn't connect to any resolved address
2025-11-11T17:18:22.160352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518391871504429:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6585) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:22.160393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518396166471820:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:22.160416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518391871504429:2049] [node 2] ICP30 unregister session Session# [4:7571518396166471820:2049] VirtualId# [4:7571518396166471805:0]
2025-11-11T17:18:22.160449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:22.160476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518391871504429:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6585) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:22.160487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6585) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:22.165104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518391871504429:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:22.241804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:22.242087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518397121338676:2050] [node 3] ICS07 socket disconnect 116 reason# EndOfStream
2025-11-11T17:18:22.242127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518397121338676:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.242216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:22.242256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518397121338676:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:22.242726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:22.242912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518396166471822:2050] [node 3] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-11T17:18:22.242936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518396166471822:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.242979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:22.243019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518396166471822:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:22.243862Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address
2025-11-11T17:18:22.244073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518391871504430:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:22.244116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518396166471822:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:22.244145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518391871504430:2050] [node 3] ICP30 unregister session Session# [4:7571518396166471822:2050] VirtualId# [4:7571518396166471808:0]
2025-11-11T17:18:22.244182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:22.244211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518391871504430:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:22.244224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:22.244611Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address
2025-11-11T17:18:22.244799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518392826371294:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:22.244833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518397121338676:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:22.244858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518392826371294:2050] [node 3] ICP30 unregister session Session# [5:7571518397121338676:2050] VirtualId# [5:7571518397121338666:0]
2025-11-11T17:18:22.244887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:22.244906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518392826371294:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:22.244917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:22.245681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:22.245893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518397233154858:2050] [node 3] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:18:22.245923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518392938187471:2050] [node 3] ICP30 unregister session Session# [6:7571518397233154858:2050] VirtualId# [6:7571518397233154852:0]
2025-11-11T17:18:22.245946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518397233154858:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.246016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:22.246050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:22.253732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518392826371294:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:22.257676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518391871504430:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:22.453764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:22.454010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518397233154850:2059] [node 4] ICS01 socket: 105 reason# EndOfStream
2025-11-11T17:18:22.454041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518392938187482:2059] [node 4] ICP30 unregister session Session# [6:7571518397233154850:2059] VirtualId# [6:7571518397233154847:0]
2025-11-11T17:18:22.454072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518397233154850:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.454144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:22.454179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:22.457735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:22.457968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518397121338687:2059] [node 4] ICS01 socket: 84 reason# EndOfStream
2025-11-11T17:18:22.457996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518397121338623:2059] [node 4] ICP30 unregister session Session# [5:7571518397121338687:2059] VirtualId# [5:7571518397121338682:0]
2025-11-11T17:18:22.458022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518397121338687:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.458088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:22.458130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:22.605777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:22.606074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518397233154844:2062] [node 5] ICS01 socket: 114 reason# EndOfStream
2025-11-11T17:18:22.606122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518392938187488:2062] [node 5] ICP30 unregister session Session# [6:7571518397233154844:2062] VirtualId# [6:7571518397233154842:0]
2025-11-11T17:18:22.606153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518397233154844:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:22.606191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:22.606286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:22.953827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518399212462660:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:22.953875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518399212462661:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:23.034899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518402152862201:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:23.034943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518402152862202:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:23.151464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518405123147220:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:23.151503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518405123147221:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:23.209894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518404102798827:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:23.209930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518404102798828:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:23.210268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518404102798826:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:23.281456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518402150067114:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:23.281494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518402150067115:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:23.281517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518402150067116:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:23.410618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518403507429968:2058] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.413051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518404102798837:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.452601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518405123147231:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29016
2025-11-11T17:18:23.539272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518403507429979:2065] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.474439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518402815143882:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:23.474476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518402815143883:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:23.474536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518402815143884:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:23.541679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518402815143899:2061] [node 4] ICP01 ready to work
2025-11-11T17:18:23.563930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518402815143884:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(40690571ns) {Configure:104(40680872ns) {ProcessConfigured:135(68240ns) {StartInitialHandshake:157(66286ns) {DropHandshakes:469(4150ns) {DropIncomingHandshake:441(774ns) DropOutgoingHandshake:456(595ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:476(2346ns) {SwitchToState:178(950ns)}}}}}
2025-11-11T17:18:23.564344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518402815143905:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.568317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518403507429983:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47113
2025-11-11T17:18:23.567288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518402815143899:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25403504ns) {Configure:104(25400184ns) {ProcessConfigured:135(36334ns) {StartInitialHandshake:157(35041ns) {DropHandshakes:469(1660ns) {DropIncomingHandshake:441(331ns) DropOutgoingHandshake:456(295ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:476(1063ns) {SwitchToState:178(412ns)}}}}}
2025-11-11T17:18:23.570167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518403507429986:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:23.522580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518405123147236:2063] [node 5] ICP01 ready to work
2025-11-11T17:18:23.707281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518402150067139:2070] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.714225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518405123147270:2073] [node 4] ICP01 ready to work
2025-11-11T17:18:23.714431Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147231:2059] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:23.714757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518405123147239:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29016
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.716945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518405123147268:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29016
2025-11-11T17:18:23.717003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518405123147269:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29016
2025-11-11T17:18:23.485981Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518404102798837:2058] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.549535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518402152862220:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21855
2025-11-11T17:18:23.702300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518404102798850:2066] [node 5] ICP01 ready to work
2025-11-11T17:18:23.771715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518402150067131:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:23.773636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518402152862243:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:23.740839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518404102798884:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
2025-11-11T17:18:23.740906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518404102798885:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
2025-11-11T17:18:23.805626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518404102798871:2074] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.838964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518404102798826:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:23.851138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518404102798903:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
2025-11-11T17:18:23.845903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518402815143883:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(370842921ns) {Configure:104(370836948ns) {ProcessConfigured:135(20016385ns) {StartInitialHandshake:157(20013775ns) {DropHandshakes:469(3233ns) {DropIncomingHandshake:441(839ns) DropOutgoingHandshake:456(645ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:476(4344ns) {SwitchToState:178(1812ns)}}}}}
2025-11-11T17:18:23.792889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518405123147255:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:23.804353Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518402152862220:2066] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:23.857913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518404102798826:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(93301665ns) {Configure:104(93298920ns) {ProcessConfigured:135(225456ns) {StartInitialHandshake:157(83688ns) {DropHandshakes:469(3729ns) {DropIncomingHandshake:441(1088ns) DropOutgoingHandshake:456(491ns)} GenerateSessionVirtualId:988(5422ns) PrepareNewSessionHandshake:476(1918ns) {SwitchToState:178(607ns)}} IncomingHandshake:261(115670ns) {DropIncomingHandshake:441(173ns) PrepareNewSessionHandshake:476(353ns) {SwitchToState:178(202ns)} GenerateSessionVirtualId:988(200ns) IssueIncomingHandshakeReply:193(85486ns) {SwitchToState:178(708ns)}}}}}
2025-11-11T17:18:23.850030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507429970:2059] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:23.849955Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147269:2095] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:23.881672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518402152862269:2061] [node 5] ICP01 ready to work
2025-11-11T17:18:23.850071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518403507429970:2059] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.920209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518402150067116:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(174223087ns) {Configure:104(174217402ns) {ProcessConfigured:135(47784ns) {StartInitialHandshake:157(45757ns) {DropHandshakes:469(1825ns) {DropIncomingHandshake:441(442ns) DropOutgoingHandshake:456(104ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:476(1331ns) {SwitchToState:178(697ns)}}}}}
2025-11-11T17:18:23.882319Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518403507429970:2059] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:23.914269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518405123147236:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(391401731ns) {Configure:104(391398231ns) {ProcessConfigured:135(69270ns) {StartInitialHandshake:157(67172ns) {DropHandshakes:469(6991ns) {DropIncomingHandshake:441(640ns) DropOutgoingHandshake:456(434ns)} GenerateSessionVirtualId:988(1709ns) PrepareNewSessionHandshake:476(1831ns) {SwitchToState:178(486ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.883596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518402152862270:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.906454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518402815143905:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(340730337ns) {Configure:104(340728619ns) {ProcessConfigured:135(67394ns) {StartInitialHandshake:157(65273ns) {DropHandshakes:469(3600ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(1018ns)} GenerateSessionVirtualId:988(902ns) PrepareNewSessionHandshake:476(1859ns) {SwitchToState:178(572ns)}}}}}
2025-11-11T17:18:23.916511Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518404102798837:2058] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:23.958300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518403507430037:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47113
2025-11-11T17:18:23.958438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518403507430038:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47113
2025-11-11T17:18:23.958513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518403507430039:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47113
2025-11-11T17:18:23.921818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518402150067184:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:23.918432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518402152862201:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(504535084ns) {Configure:104(504532188ns) {ProcessConfigured:135(73591ns) {StartInitialHandshake:157(71419ns) {DropHandshakes:469(3154ns) {DropIncomingHandshake:441(618ns) DropOutgoingHandshake:456(662ns)} GenerateSessionVirtualId:988(497ns) PrepareNewSessionHandshake:476(1359ns) {SwitchToState:178(387ns)}}}}}
2025-11-11T17:18:23.914413Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147239:2066] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:23.935036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518404102798826:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:23.948795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518405123147301:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29016
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.942715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.033957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111257:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:23.982140Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:24.040512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518402152862269:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(134526745ns) {Configure:104(134520497ns) {ProcessConfigured:135(67030ns) {StartInitialHandshake:157(65114ns) {DropHandshakes:469(2233ns) {DropIncomingHandshake:441(328ns) DropOutgoingHandshake:456(704ns)} GenerateSessionVirtualId:988(1295ns) PrepareNewSessionHandshake:476(1061ns) {SwitchToState:178(244ns)}}}}}
2025-11-11T17:18:23.970208Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147268:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:23.990118Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507429983:2067] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.016345Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507430039:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.029771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518403507429968:2058] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:24.034498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518403507429968:2058] [node 4] ICP35 event processing took too much time PendingConnection:278(44161556ns) {HandleHandshakeStatus:395(16157327ns) {LogHandshakeFail:472(9464809ns) {UpdateErrorStateLog:544(5483052ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.942772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518404102798826:2048] [node 1] ICP22 created new session: [4:7571518404102798905:2048]
2025-11-11T17:18:24.023432Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518402150067184:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.021765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518405123147220:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(391129489ns) {Configure:104(391123448ns) {ProcessConfigured:135(27095117ns) {StartInitialHandshake:157(27092622ns) {DropHandshakes:469(3136ns) {DropIncomingHandshake:441(806ns) DropOutgoingHandshake:456(527ns)} GenerateSessionVirtualId:988(1812ns) PrepareNewSessionHandshake:476(3244ns) {SwitchToState:178(738ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.034550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518403507429968:2058] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.045804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518407110111257:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:23.942819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518404102798905:2048] [node 1] ICS09 handshake done sender: [4:7571518404102798837:2058] self: [4:7571518404102798902:0] peer: [1:7571518403507429969:0] socket: 87 qp: -1
2025-11-11T17:18:24.057232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518402152862243:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(222154518ns) {Configure:104(222144797ns) {ProcessConfigured:135(15193931ns) {StartInitialHandshake:157(11445901ns) {DropHandshakes:469(2779ns) {DropIncomingHandshake:441(457ns) DropOutgoingHandshake:456(270ns)} GenerateSessionVirtualId:988(2438ns) PrepareNewSessionHandshake:476(2746ns) {SwitchToState:178(493ns)}}}}}
2025-11-11T17:18:24.023376Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147301:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.037858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518402150067179:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.034594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.942852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518404102798905:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:24.049316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518402815143882:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.030382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518405123147220:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.057531Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518402152862227:2072] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.041649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518402150067116:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.057572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518402152862227:2072] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.034638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518403507429968:2058] [node 4] ICP22 created new session: [1:7571518407802397339:2058]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.942886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.089214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518402815143960:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:23.979334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518404102798885:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.035127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518405123147255:2069] [node 6] ICP35 event processing took too much time PendingConnection:278(33880846ns) {IncomingHandshake:261(865868ns) {DropIncomingHandshake:441(832ns) PrepareNewSessionHandshake:476(2168ns) {SwitchToState:178(743ns)} GenerateSessionVirtualId:988(1006ns) IssueIncomingHandshakeReply:193(4732ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.130089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518402815143960:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.034707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518407802397339:2058] [node 4] ICS09 handshake done sender: [1:7571518403507429970:2059] self: [1:7571518403507429969:0] peer: [4:7571518404102798902:0] socket: 86 qp: -1
2025-11-11T17:18:24.050786Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518402150067179:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.072436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518402152862270:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(109148079ns) {Configure:104(109144521ns) {ProcessConfigured:135(115446ns) {StartInitialHandshake:157(72916ns) {DropHandshakes:469(3339ns) {DropIncomingHandshake:441(645ns) DropOutgoingHandshake:456(674ns)} GenerateSessionVirtualId:988(4330ns) PrepareNewSessionHandshake:476(2187ns) {SwitchToState:178(732ns)}} IncomingHandshake:261(18764ns) {DropIncomingHandshake:441(148ns) PrepareNewSessionHandshake:476(215ns) {SwitchToState:178(128ns)} GenerateSessionVirtualId:988(66ns) IssueIncomingHandshakeReply:193(895ns)}}}}
2025-11-11T17:18:24.041265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518405123147221:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(556785223ns) {Configure:104(556779021ns) {ProcessConfigured:135(32657469ns) {StartInitialHandshake:157(32653942ns) {DropHandshakes:469(3235ns) {DropIncomingHandshake:441(748ns) DropOutgoingHandshake:456(647ns)} GenerateSessionVirtualId:988(391ns) PrepareNewSessionHandshake:476(3576ns) {SwitchToState:178(1292ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.085147Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518402152862227:2072] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:24.023428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518404102798828:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.034771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518407802397339:2058] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.041395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518405123147221:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.080410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518406445034485:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.148563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518402815143905:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.104848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518406447829579:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21855
2025-11-11T17:18:24.041948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518404102798828:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(25236956ns) {IncomingHandshake:261(25234605ns) {DropIncomingHandshake:441(864ns) PrepareNewSessionHandshake:476(2110ns) {SwitchToState:178(727ns)} GenerateSessionVirtualId:988(1391ns) IssueIncomingHandshakeReply:193(5637469ns) {SwitchToState:178(1277ns)}}}
2025-11-11T17:18:24.042625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518404102798850:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(268694100ns) {Configure:104(239575081ns) {ProcessConfigured:135(692063ns) {StartInitialHandshake:157(689613ns) {DropHandshakes:469(1291ns) {DropIncomingHandshake:441(223ns) DropOutgoingHandshake:456(168ns)} GenerateSessionVirtualId:988(350ns) PrepareNewSessionHandshake:476(1682ns) {SwitchToState:178(716ns)}}}}}
2025-11-11T17:18:24.048891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518408397766206:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
2025-11-11T17:18:24.056072Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518404102798884:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.056570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518404102798826:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(121359103ns) {HandleHandshakeStatus:329(121356086ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(6306ns) SwitchToState:178(791ns) ProcessPendingSessionEvents:495(40082819ns) {ForwardSessionEventToSession:612(6658278ns) ForwardSessionEventToSession:612(7284598ns) ForwardSessionEventToSession:612(7309ns) ForwardSessionEventToSession:612(9214464ns) ForwardSessionEventToSession:612(3858ns) ForwardSessionEventToSession:612(2689052ns) ForwardSessionEventToSession:612(3956ns) ForwardSessionEventToSession:612(2351ns) ForwardSessionEventToSession:612(6195274ns) ForwardSessionEventToSession:612(7999447ns)}}}
2025-11-11T17:18:24.105900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518408397766208:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
2025-11-11T17:18:24.106748Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518404102798885:2102] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:24.106962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518404102798828:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.107038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:24.107078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518404102798828:2050] [node 3] ICP22 created new session: [4:7571518408397766209:2050]
2025-11-11T17:18:24.107120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518408397766209:2050] [node 3] ICS09 handshake done sender: [4:7571518404102798885:2102] self: [4:7571518404102798907:0] peer: [3:7571518405123147272:0] socket: 108 qp: -1
2025-11-11T17:18:24.107152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518408397766209:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:24.107194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:24.122825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518404102798871:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(284983260ns) {Configure:104(284975187ns) {ProcessConfigured:135(24794766ns) {StartInitialHandshake:157(8960632ns) {DropHandshakes:469(4010ns) {DropIncomingHandshake:441(1022ns) DropOutgoingHandshake:456(553ns)} GenerateSessionVirtualId:988(2137ns) PrepareNewSessionHandshake:476(3707ns) {SwitchToState:178(1937ns)}}}}}
2025-11-11T17:18:24.147124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518408397766206:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.147247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518404102798827:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.159713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518408397766205:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.159735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518408397766205:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.160598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518408397766216:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
2025-11-11T17:18:24.161003Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518408397766205:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:24.173768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518408397766206:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:24.175891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518404102798827:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.175928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:24.175965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518404102798827:2049] [node 2] ICP22 created new session: [4:7571518408397766218:2049]
2025-11-11T17:18:24.176005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518408397766218:2049] [node 2] ICS09 handshake done sender: [4:7571518408397766206:2118] self: [4:7571518408397766215:0] peer: [2:7571518406447829570:0] socket: 125 qp: -1
2025-11-11T17:18:24.176032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518408397766218:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:24.176076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:24.193786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518404102798850:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.217524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:24.221710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518404102798850:2066] [node 5] ICP22 created new session: [4:7571518408397766220:2066]
2025-11-11T17:18:24.230633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518408397766221:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15962
2025-11-11T17:18:24.062133Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147273:2097] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.062149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518405123147273:2097] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.085331Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147297:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.089011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518409418114606:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29016
2025-11-11T17:18:24.089468Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518405123147273:2097] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:24.097728Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518405123147297:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.125972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518409418114607:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29016
2025-11-11T17:18:24.143006Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518405123147297:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:24.148873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518405123147231:2059] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:24.153747Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518405123147269:2095] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:24.157832Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:24.175638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518405123147270:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.175693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:24.175744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518405123147270:2073] [node 4] ICP22 created new session: [3:7571518409418114609:2073]
2025-11-11T17:18:24.182142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518409418114609:2073] [node 4] ICS09 handshake done sender: [3:7571518405123147273:2097] self: [3:7571518405123147272:0] peer: [4:7571518404102798907:0] socket: 107 qp: -1
2025-11-11T17:18:24.201798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518405123147220:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.201857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:24.201901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518405123147220:2048] [node 1] ICP22 created new session: [3:7571518409418114610:2048]
2025-11-11T17:18:24.201944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518409418114610:2048] [node 1] ICS09 handshake done sender: [3:7571518405123147231:2059] self: [3:7571518409418114602:0] peer: [1:7571518403507429973:0] socket: 89 qp: -1
2025-11-11T17:18:24.201980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518409418114610:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:24.202022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:24.202281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518405123147220:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:24.202613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518405123147236:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.202654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:24.202688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518405123147236:2063] [node 5] ICP22 created new session: [3:7571518409418114612:2063]
2025-11-11T17:18:24.202741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518409418114612:2063] [node 5] ICS09 handshake done sender: [3:7571518405123147297:2116] self: [3:7571518405123147296:0] peer: [5:7571518406445034483:0] socket: 113 qp: -1
2025-11-11T17:18:24.202768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518409418114612:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:24.202792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:24.203046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518405123147221:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.203074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:24.203102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518405123147221:2049] [node 2] ICP22 created new session: [3:7571518409418114614:2049]
2025-11-11T17:18:24.203137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518409418114614:2049] [node 2] ICS09 handshake done sender: [3:7571518405123147269:2095] self: [3:7571518409418114604:0] peer: [2:7571518402152862226:0] socket: 106 qp: -1
2025-11-11T17:18:24.203159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518409418114614:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:24.203184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:24.204516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518409418114609:2073] [node 4] ICS10 traffic start
2025-11-11T17:18:24.207072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:24.254651Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518408397766212:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.254680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518408397766212:2120] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.034816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.262489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518405123147270:2073] [node 4] ICP35 event processing took too much time PendingConnection:278(86668436ns) {HandleHandshakeStatus:329(86666373ns) {DropIncomingHandshake:441(13227ns) DropOutgoingHandshake:456(583ns) SwitchToState:178(764ns) ProcessPendingSessionEvents:495(5995563ns) {ForwardSessionEventToSession:612(5947492ns) ForwardSessionEventToSession:612(8522ns) ForwardSessionEventToSession:612(2157ns) ForwardSessionEventToSession:612(4797ns) ForwardSessionEventToSession:612(2082ns) ForwardSessionEventToSession:612(2314ns) ForwardSessionEventToSession:612(4352ns) ForwardSessionEventToSession:612(4941ns) ForwardSessionEventToSession:612(2229ns) ForwardSessionEventToSession:612(6650ns)}}}
2025-11-11T17:18:24.117685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518402152862202:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.182500Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:24.122422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518406445034488:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.142247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518402150067114:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(317235343ns) {Configure:104(317222128ns) {ProcessConfigured:135(43623ns) {StartInitialHandshake:157(41269ns) {DropHandshakes:469(2086ns) {DropIncomingHandshake:441(268ns) DropOutgoingHandshake:456(628ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:476(1499ns) {SwitchToState:178(377ns)}}}}}
2025-11-11T17:18:24.142418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518402150067114:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.143127Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518406445034485:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.158896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518402150067139:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(387026593ns) {Configure:104(387018279ns) {ProcessConfigured:135(20455616ns) {StartInitialHandshake:157(20452593ns) {DropHandshakes:469(2097ns) {DropIncomingHandshake:441(244ns) DropOutgoingHandshake:456(544ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:476(6173ns) {SwitchToState:178(809ns)}}}}}
2025-11-11T17:18:24.159096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518402150067139:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.161663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518402150067183:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.161691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518402150067183:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.170206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518406445034495:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.174463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518406445034497:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.204085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518406445034496:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.214823Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518402150067184:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:24.215116Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518402150067179:2110] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:24.215368Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518406445034485:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:24.217653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518402150067115:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(260298361ns) {Configure:104(248161580ns) {ProcessConfigured:135(42563444ns) {StartInitialHandshake:157(42560460ns) {DropHandshakes:469(1125ns) {DropIncomingHandshake:441(263ns) DropOutgoingHandshake:456(165ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:476(3020ns) {SwitchToState:178(1191ns)}}}}}
2025-11-11T17:18:24.226054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518402150067183:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:24.226291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518402150067116:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.226342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:24.226395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518402150067116:2050] [node 3] ICP22 created new session: [5:7571518406445034499:2050]
2025-11-11T17:18:24.226441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518406445034499:2050] [node 3] ICS09 handshake done sender: [5:7571518402150067184:2113] self: [5:7571518406445034483:0] peer: [3:7571518405123147296:0] socket: 115 qp: -1
2025-11-11T17:18:24.226474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518406445034499:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:24.226507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:24.228019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518402150067114:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.228058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:24.228090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518402150067114:2048] [node 1] ICP22 created new session: [5:7571518406445034501:2048]
2025-11-11T17:18:24.228122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518406445034501:2048] [node 1] ICS09 handshake done sender: [5:7571518402150067179:2110] self: [5:7571518406445034492:0] peer: [1:7571518403507429980:0] socket: 109 qp: -1
2025-11-11T17:18:24.228156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518406445034501:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:24.228206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:24.229624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518402150067139:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.229658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:24.229697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518402150067139:2070] [node 4] ICP22 created new session: [5:7571518406445034505:2070]
2025-11-11T17:18:24.229741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518406445034505:2070] [node 4] ICS09 handshake done sender: [5:7571518406445034485:2114] self: [5:7571518406445034494:0] peer: [4:7571518408397766204:0] socket: 127 qp: -1
2025-11-11T17:18:24.229769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518406445034505:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:24.229820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:24.249674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518406445034496:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.256423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518402150067115:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.263465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518402150067131:2059] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.273726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:24.273834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518402150067131:2059] [node 6] ICP22 created new session: [5:7571518406445034508:2059]
2025-11-11T17:18:24.273900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518406445034508:2059] [node 6] ICS09 handshake done sender: [5:7571518402150067183:2112] self: [5:7571518402150067182:0] peer: [6:7571518407110111260:0] socket: 117 qp: -1
2025-11-11T17:18:24.273937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518406445034508:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:24.273983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:24.042016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507430038:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.117773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:24.061872Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507429974:2061] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.061912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518403507429974:2061] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.095846Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518403507429974:2061] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:24.126240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518399212462661:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.126317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:24.126363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518399212462661:2049] [node 3] ICP22 created new session: [1:7571518407802397343:2049]
2025-11-11T17:18:24.126404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518407802397343:2049] [node 3] ICS09 handshake done sender: [1:7571518403507429974:2061] self: [1:7571518403507429973:0] peer: [3:7571518409418114602:0] socket: 88 qp: -1
2025-11-11T17:18:24.126434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518407802397343:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:24.126471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:24.134383Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507429988:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.134416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518403507429988:2070] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.146252Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507429981:2066] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.146276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518403507429981:2066] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.149558Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518403507429981:2066] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:24.150912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518403507429979:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.150958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:24.150999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518403507429979:2065] [node 5] ICP22 created new session: [1:7571518407802397347:2065]
2025-11-11T17:18:24.151040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518407802397347:2065] [node 5] ICS09 handshake done sender: [1:7571518403507429981:2066] self: [1:7571518403507429980:0] peer: [5:7571518406445034492:0] socket: 94 qp: -1
2025-11-11T17:18:24.151072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518407802397347:2065] [node 5] ICS10 traffic start
2025-11-11T17:18:24.151112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:24.153785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518403507429998:2079] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.170574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518403507429998:2079] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.170730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518399212462660:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(50308613ns) {IncomingHandshake:261(23985189ns) {DropIncomingHandshake:441(796ns) PrepareNewSessionHandshake:476(1941ns) {SwitchToState:178(899ns)} GenerateSessionVirtualId:988(915ns) IssueIncomingHandshakeReply:193(11015ns)}}
2025-11-11T17:18:24.175685Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518403507429988:2070] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:24.178719Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518403507429998:2079] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:24.178823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518403507429986:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.178874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:24.178914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518403507429986:2069] [node 6] ICP22 created new session: [1:7571518407802397351:2069]
2025-11-11T17:18:24.178961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518407802397351:2069] [node 6] ICS09 handshake done sender: [1:7571518403507429988:2070] self: [1:7571518403507429987:0] peer: [6:7571518407110111258:0] socket: 112 qp: -1
2025-11-11T17:18:24.178998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518407802397351:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:24.179032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:24.180326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518399212462660:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.180369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:24.180405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518399212462660:2048] [node 2] ICP22 created new session: [1:7571518407802397353:2048]
2025-11-11T17:18:24.180445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518407802397353:2048] [node 2] ICS09 handshake done sender: [1:7571518403507429998:2079] self: [1:7571518403507429997:0] peer: [2:7571518406447829583:0] socket: 139 qp: -1
2025-11-11T17:18:24.180472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518407802397353:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:24.180502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:24.183401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518402815143882:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518407110111257:2116] held: no
2025-11-11T17:18:24.183887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111261:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.184647Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518407110111257:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:24.184848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518402815143882:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.184900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:24.184930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518402815143882:2048] [node 1] ICP22 created new session: [6:7571518407110111262:2048]
2025-11-11T17:18:24.184969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518407110111262:2048] [node 1] ICS09 handshake done sender: [6:7571518407110111257:2116] self: [6:7571518407110111258:0] peer: [1:7571518403507429987:0] socket: 119 qp: -1
2025-11-11T17:18:24.184991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518407110111262:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:24.185023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:24.210183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111265:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.245924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111267:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.250590Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518407110111265:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.250706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518402815143883:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.253745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111259:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.254259Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518407110111259:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.254368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518402815143899:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.255791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111270:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.262116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111271:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.285748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111266:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.263455Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:24.271292Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518408397766212:2120] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:24.117823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518402152862202:2049] [node 3] ICP22 created new session: [2:7571518406447829580:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.318154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518406445034510:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.310344Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:24.277734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518408397766220:2066] [node 5] ICS09 handshake done sender: [4:7571518408397766205:2117] self: [4:7571518408397766204:0] peer: [5:7571518406445034494:0] socket: 126 qp: -1
2025-11-11T17:18:24.278209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518408397766220:2066] [node 5] ICS10 traffic start
2025-11-11T17:18:24.285988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:24.286478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518404102798850:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(110960528ns) {HandleHandshakeStatus:329(110957217ns) {DropIncomingHandshake:441(2215ns) DropOutgoingHandshake:456(3917ns) SwitchToState:178(536ns) ProcessPendingSessionEvents:495(95986ns) {ForwardSessionEventToSession:612(50689ns) ForwardSessionEventToSession:612(5396ns) ForwardSessionEventToSession:612(4311ns) ForwardSessionEventToSession:612(5174ns) ForwardSessionEventToSession:612(2336ns) ForwardSessionEventToSession:612(4232ns) ForwardSessionEventToSession:612(1716ns) ForwardSessionEventToSession:612(5138ns) ForwardSessionEventToSession:612(2158ns) ForwardSessionEventToSession:612(4689ns)}}}
2025-11-11T17:18:24.313881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518404102798871:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.313934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:24.313987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518404102798871:2074] [node 6] ICP22 created new session: [4:7571518408397766224:2074]
2025-11-11T17:18:24.314029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518408397766224:2074] [node 6] ICS09 handshake done sender: [4:7571518408397766212:2120] self: [4:7571518408397766211:0] peer: [6:7571518407110111269:0] socket: 137 qp: -1
2025-11-11T17:18:24.314067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518408397766224:2074] [node 6] ICS10 traffic start
2025-11-11T17:18:24.314096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:24.311603Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518407110111265:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:24.321681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518407110111259:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:24.326548Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518402815143960:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:24.330517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518402815143883:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518407110111265:2119] held: no
2025-11-11T17:18:24.117873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518406447829580:2049] [node 3] ICS09 handshake done sender: [2:7571518402152862227:2072] self: [2:7571518402152862226:0] peer: [3:7571518409418114604:0] socket: 103 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:24.117914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518406447829580:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:24.337302Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518405123147294:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.117962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:24.127741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518406447829582:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21855
2025-11-11T17:18:24.133907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518406447829582:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.134052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518402152862201:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.141859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518406447829579:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.147621Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518406447829571:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.147637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518406447829571:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.159495Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518406447829571:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:24.161259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518402152862270:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.161314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:24.161356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518402152862270:2063] [node 4] ICP22 created new session: [2:7571518406447829587:2063]
2025-11-11T17:18:24.161399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518406447829587:2063] [node 4] ICS09 handshake done sender: [2:7571518406447829571:2113] self: [2:7571518406447829570:0] peer: [4:7571518408397766215:0] socket: 124 qp: -1
2025-11-11T17:18:24.161432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518406447829587:2063] [node 4] ICS10 traffic start
2025-11-11T17:18:24.161467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:24.178346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518406447829589:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21855
2025-11-11T17:18:24.180007Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518406447829582:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:24.181285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518402152862201:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.181323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:24.181363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518402152862201:2048] [node 1] ICP22 created new session: [2:7571518406447829590:2048]
2025-11-11T17:18:24.181400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518406447829590:2048] [node 1] ICS09 handshake done sender: [2:7571518406447829582:2117] self: [2:7571518406447829583:0] peer: [1:7571518403507429997:0] socket: 140 qp: -1
2025-11-11T17:18:24.181424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518406447829590:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:24.181455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:24.216220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518406447829593:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21855
2025-11-11T17:18:24.249652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518406447829593:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.257706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518406447829577:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.257730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518406447829577:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.275347Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518406447829577:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:24.281632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518406447829575:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.281674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518406447829575:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.289757Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518406447829575:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:24.294053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518402152862243:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.294158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:24.294202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518402152862243:2059] [node 6] ICP22 created new session: [2:7571518406447829597:2059]
2025-11-11T17:18:24.294253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518406447829597:2059] [node 6] ICS09 handshake done sender: [2:7571518406447829577:2115] self: [2:7571518406447829576:0] peer: [6:7571518407110111268:0] socket: 97 qp: -1
2025-11-11T17:18:24.294288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518406447829597:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:24.294333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:24.313669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518402152862269:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.320736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:24.320795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518402152862269:2061] [node 5] ICP22 created new session: [2:7571518406447829599:2061]
2025-11-11T17:18:24.320843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518406447829599:2061] [node 5] ICS09 handshake done sender: [2:7571518406447829575:2114] self: [2:7571518406447829574:0] peer: [5:7571518406445034507:0] socket: 141 qp: -1
2025-11-11T17:18:24.320883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518406447829599:2061] [node 5] ICS10 traffic start
2025-11-11T17:18:24.320937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:24.326214Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518406445034496:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:24.341774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518402150067115:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.343993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:24.344052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518402150067115:2049] [node 2] ICP22 created new session: [5:7571518406445034511:2049]
2025-11-11T17:18:24.344101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518406445034511:2049] [node 2] ICS09 handshake done sender: [5:7571518406445034496:2120] self: [5:7571518406445034507:0] peer: [2:7571518406447829574:0] socket: 146 qp: -1
2025-11-11T17:18:24.337331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518405123147294:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:24.344143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518406445034511:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:24.344197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:24.354400Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518405123147294:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:24.345934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518406445034513:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50234
2025-11-11T17:18:24.358895Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:24.334066Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23102) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:24.359731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518405123147255:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:24.334927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518402815143899:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.359482Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:24.390403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:24.390478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518405123147255:2069] [node 6] ICP22 created new session: [3:7571518409418114620:2069]
2025-11-11T17:18:24.390523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518409418114620:2069] [node 6] ICS09 handshake done sender: [3:7571518405123147294:2115] self: [3:7571518405123147293:0] peer: [6:7571518407110111277:0] socket: 152 qp: -1
2025-11-11T17:18:24.390564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518409418114620:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:24.390611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:24.391135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518405123147255:2069] [node 6] ICP35 event processing took too much time PendingConnection:278(31185279ns) {HandleHandshakeStatus:329(31182057ns) {DropIncomingHandshake:441(19397ns) DropOutgoingHandshake:456(672ns) SwitchToState:178(1100ns) ProcessPendingSessionEvents:495(101421ns) {ForwardSessionEventToSession:612(61483ns) ForwardSessionEventToSession:612(2669ns) ForwardSessionEventToSession:612(2760ns) ForwardSessionEventToSession:612(5081ns) ForwardSessionEventToSession:612(1962ns) ForwardSessionEventToSession:612(4427ns) ForwardSessionEventToSession:612(4253ns) ForwardSessionEventToSession:612(2772ns) ForwardSessionEventToSession:612(5384ns) ForwardSessionEventToSession:612(2568ns)}}}
2025-11-11T17:18:24.359802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518402150067115:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:24.334995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:24.335042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518402815143899:2061] [node 4] ICP22 created new session: [6:7571518407110111273:2061]
2025-11-11T17:18:24.335106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518407110111273:2061] [node 4] ICS09 handshake done sender: [6:7571518407110111259:2117] self: [6:7571518407110111269:0] peer: [4:7571518408397766211:0] socket: 138 qp: -1
2025-11-11T17:18:24.335139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518407110111273:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:24.335196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:24.335436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518402815143899:2061] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:24.335707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518402815143905:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.335738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:24.335768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518402815143905:2066] [node 5] ICP22 created new session: [6:7571518407110111275:2066]
2025-11-11T17:18:24.335798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518407110111275:2066] [node 5] ICS09 handshake done sender: [6:7571518402815143960:2115] self: [6:7571518407110111260:0] peer: [5:7571518402150067182:0] socket: 118 qp: -1
2025-11-11T17:18:24.335820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518407110111275:2066] [node 5] ICS10 traffic start
2025-11-11T17:18:24.335887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:24.336954Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518407110111267:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:24.337070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518402815143884:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:24.350298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518407110111278:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56125
2025-11-11T17:18:24.363799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518402815143883:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(42672321ns) {HandleHandshakeStatus:395(42669302ns) {LogHandshakeFail:472(12275ns) {UpdateErrorStateLog:544(4264ns)} DropOutgoingHandshake:456(1461ns) ProcessPendingSessionEvents:495(23644855ns) {DropSessionEvent:512(15207ns) DropSessionEvent:512(12873ns) DropSessionEvent:512(2965ns) DropSessionEvent:512(3644ns) DropSessionEvent:512(2996ns) DropSessionEvent:512(3275ns) DropSessionEvent:512(3202ns) DropSessionEvent:512(3422ns) DropSessionEvent:512(3605ns) DropSessionEvent:512(7808158ns)}}}
2025-11-11T17:18:24.363864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518402815143883:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.363947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:24.363997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518402815143883:2049] [node 2] ICP22 created new session: [6:7571518407110111279:2049]
2025-11-11T17:18:24.364034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518407110111279:2049] [node 2] ICS09 handshake done sender: [6:7571518407110111265:2119] self: [6:7571518407110111268:0] peer: [2:7571518406447829576:0] socket: 102 qp: -1
2025-11-11T17:18:24.364071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518407110111279:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:24.364115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:24.365101Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518407110111267:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:24.365294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518402815143884:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:24.379333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:24.379391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518402815143884:2050] [node 3] ICP22 created new session: [6:7571518407110111281:2050]
2025-11-11T17:18:24.379439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518407110111281:2050] [node 3] ICS09 handshake done sender: [6:7571518407110111267:2121] self: [6:7571518407110111277:0] peer: [3:7571518405123147293:0] socket: 153 qp: -1
2025-11-11T17:18:24.379475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518407110111281:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:24.379521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:24.522846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:24.523085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518406447829590:2048] [node 1] ICS01 socket: 140 reason# EndOfStream
2025-11-11T17:18:24.523121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518402152862201:2048] [node 1] ICP30 unregister session Session# [2:7571518406447829590:2048] VirtualId# [2:7571518406447829583:0]
2025-11-11T17:18:24.523154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518406447829590:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.523225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:24.523261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:24.529908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:24.530126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518409418114610:2048] [node 1] ICS07 socket disconnect 89 reason# EndOfStream
2025-11-11T17:18:24.530148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518409418114610:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.530244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:24.530288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518409418114610:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:24.531223Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address
2025-11-11T17:18:24.531463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518405123147220:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:24.531502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518409418114610:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:24.531535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518405123147220:2048] [node 1] ICP30 unregister session Session# [3:7571518409418114610:2048] VirtualId# [3:7571518409418114602:0]
2025-11-11T17:18:24.531575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:24.531600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518405123147220:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:24.531631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:24.533756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:24.533949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518404102798905:2048] [node 1] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-11T17:18:24.533970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518404102798905:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.534030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:24.534066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518404102798905:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:24.534865Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address
2025-11-11T17:18:24.535105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518404102798826:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:24.535160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518404102798905:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:24.535192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518404102798826:2048] [node 1] ICP30 unregister session Session# [4:7571518404102798905:2048] VirtualId# [4:7571518404102798902:0]
2025-11-11T17:18:24.535231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:24.535262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518404102798826:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:24.535277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:24.535962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:24.536156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518406445034501:2048] [node 1] ICS01 socket: 109 reason# EndOfStream
2025-11-11T17:18:24.536182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518402150067114:2048] [node 1] ICP30 unregister session Session# [5:7571518406445034501:2048] VirtualId# [5:7571518406445034492:0]
2025-11-11T17:18:24.536204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518406445034501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.536265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:24.536299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:24.537743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:24.538003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518407110111262:2048] [node 1] ICS01 socket: 119 reason# EndOfStream
2025-11-11T17:18:24.538046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518402815143882:2048] [node 1] ICP30 unregister session Session# [6:7571518407110111262:2048] VirtualId# [6:7571518407110111258:0]
2025-11-11T17:18:24.538077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518407110111262:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.538144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:24.538181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:24.544707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518404102798826:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:24.554141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518405123147220:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:24.593769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:24.594009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518409418114614:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-11-11T17:18:24.594062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518405123147221:2049] [node 2] ICP30 unregister session Session# [3:7571518409418114614:2049] VirtualId# [3:7571518409418114604:0]
2025-11-11T17:18:24.594099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518409418114614:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.594181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:24.594216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:24.621755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:24.622004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518406445034511:2049] [node 2] ICS01 socket: 146 reason# EndOfStream
2025-11-11T17:18:24.622033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518402150067115:2049] [node 2] ICP30 unregister session Session# [5:7571518406445034511:2049] VirtualId# [5:7571518406445034507:0]
2025-11-11T17:18:24.622059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518406445034511:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.622232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:24.622495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518408397766218:2049] [node 2] ICS01 socket: 125 reason# EndOfStream
2025-11-11T17:18:24.622123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:24.622159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:24.622536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518404102798827:2049] [node 2] ICP30 unregister session Session# [4:7571518408397766218:2049] VirtualId# [4:7571518408397766215:0]
2025-11-11T17:18:24.622570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518408397766218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.622612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:24.622640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:24.629865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:24.630091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518407110111279:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-11-11T17:18:24.630553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518402815143883:2049] [node 2] ICP30 unregister session Session# [6:7571518407110111279:2049] VirtualId# [6:7571518407110111268:0]
2025-11-11T17:18:24.630585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518407110111279:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.630669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:24.630719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:24.723495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:24.723735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518406445034499:2050] [node 3] ICS01 socket: 115 reason# EndOfStream
2025-11-11T17:18:24.723772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518402150067116:2050] [node 3] ICP30 unregister session Session# [5:7571518406445034499:2050] VirtualId# [5:7571518406445034483:0]
2025-11-11T17:18:24.723799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518406445034499:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.723866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:24.723899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:24.729743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:24.729754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:24.729981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518408397766209:2050] [node 3] ICS01 socket: 108 reason# EndOfStream
2025-11-11T17:18:24.729985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518407110111281:2050] [node 3] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:18:24.730021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518402815143884:2050] [node 3] ICP30 unregister session Session# [6:7571518407110111281:2050] VirtualId# [6:7571518407110111277:0]
2025-11-11T17:18:24.730057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518407110111281:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.730132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:24.730167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:24.730014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518404102798828:2050] [node 3] ICP30 unregister session Session# [4:7571518408397766209:2050] VirtualId# [4:7571518404102798907:0]
2025-11-11T17:18:24.730040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518408397766209:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.730106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:24.730143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:24.833789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:24.834023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518406445034505:2070] [node 4] ICS01 socket: 127 reason# EndOfStream
2025-11-11T17:18:24.834058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518402150067139:2070] [node 4] ICP30 unregister session Session# [5:7571518406445034505:2070] VirtualId# [5:7571518406445034494:0]
2025-11-11T17:18:24.834090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518406445034505:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.834169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:24.834212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:24.834293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:24.834461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518407110111273:2061] [node 4] ICS07 socket disconnect 138 reason# EndOfStream
2025-11-11T17:18:24.834483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518407110111273:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.834537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:24.834572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518407110111273:2061] [node 4] ICS15 start handshake
2025-11-11T17:18:24.835891Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23102) Couldn't connect to any resolved address
2025-11-11T17:18:24.836116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518402815143899:2061] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23102) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:24.836159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518407110111273:2061] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:24.836190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518402815143899:2061] [node 4] ICP30 unregister session Session# [6:7571518407110111273:2061] VirtualId# [6:7571518407110111269:0]
2025-11-11T17:18:24.836232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:24.836267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518402815143899:2061] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23102) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:24.836281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23102) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:24.849091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518402815143899:2061] [node 4] ICP33 wake up from error state
2025-11-11T17:18:24.906104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:24.906359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518407110111275:2066] [node 5] ICS01 socket: 118 reason# EndOfStream
2025-11-11T17:18:24.906402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518402815143905:2066] [node 5] ICP30 unregister session Session# [6:7571518407110111275:2066] VirtualId# [6:7571518407110111260:0]
2025-11-11T17:18:24.906433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518407110111275:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:24.906542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:24.906591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:25.157874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518412063499722:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:25.157933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518412063499723:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:25.217819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518413616546879:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:25.217874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518413616546880:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:25.313786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518413639892412:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:25.313826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518413639892413:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:25.333897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518414609166411:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:25.333939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518414609166412:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:25.333960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518414609166413:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:25.409823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518412912014793:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:25.409885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518412912014794:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:25.409926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518412912014795:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:25.838633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518412225926179:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:25.838682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518412225926180:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:25.838743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518412225926181:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:25.825705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518414609166473:2075] [node 6] ICP01 ready to work
2025-11-11T17:18:25.869848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518413616546942:2073] [node 6] ICP01 ready to work
2025-11-11T17:18:25.893649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518412225926198:2059] [node 5] ICP01 ready to work
2025-11-11T17:18:25.872508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518414609166475:2082] [node 5] ICP01 ready to work
2025-11-11T17:18:25.905733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518412063499785:2067] [node 4] ICP01 ready to work
2025-11-11T17:18:25.942086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518412063499786:2068] [node 5] ICP01 ready to work
2025-11-11T17:18:25.929761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518412225926229:2063] [node 4] ICP01 ready to work
2025-11-11T17:18:25.960930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518413616546943:2074] [node 5] ICP01 ready to work
2025-11-11T17:18:25.974485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518413639892413:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(328419043ns) {Configure:104(316464642ns) {ProcessConfigured:135(90754ns) {StartInitialHandshake:157(74299ns) {DropHandshakes:469(3590ns) {DropIncomingHandshake:441(691ns) DropOutgoingHandshake:456(592ns)} GenerateSessionVirtualId:988(1379ns) PrepareNewSessionHandshake:476(2426ns) {SwitchToState:178(1018ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:25.975003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518413639892477:2072] [node 6] ICP01 ready to work
2025-11-11T17:18:25.958551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518412225926229:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25034325ns) {Configure:104(25030797ns) {ProcessConfigured:135(72413ns) {StartInitialHandshake:157(70060ns) {DropHandshakes:469(3536ns) {DropIncomingHandshake:441(718ns) DropOutgoingHandshake:456(655ns)} GenerateSessionVirtualId:988(1104ns) PrepareNewSessionHandshake:476(2417ns) {SwitchToState:178(1074ns)}}}}}
2025-11-11T17:18:26.021935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518414609166473:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(149085775ns) {Configure:104(149080089ns) {ProcessConfigured:135(19879229ns) {StartInitialHandshake:157(19875786ns) {DropHandshakes:469(7876594ns) {DropIncomingHandshake:441(2746ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(2871ns) PrepareNewSessionHandshake:476(4148ns) {SwitchToState:178(2019ns)}}}}}
2025-11-11T17:18:25.943913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518412063499789:2086] [node 6] ICP01 ready to work
2025-11-11T17:18:26.001925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518413616546944:2076] [node 4] ICP01 ready to work
2025-11-11T17:18:26.049845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518412063499786:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(80485640ns) {Configure:104(80480031ns) {ProcessConfigured:135(27643754ns) {StartInitialHandshake:157(27640910ns) {DropHandshakes:469(3707ns) {DropIncomingHandshake:441(839ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(848ns) PrepareNewSessionHandshake:476(4184ns) {SwitchToState:178(2209ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:25.975269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518413639892478:2074] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.074279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518418904133785:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28224
2025-11-11T17:18:26.059448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893553:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.059653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893554:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:25.975509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518413639892479:2080] [node 5] ICP01 ready to work
2025-11-11T17:18:26.081943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518416358467097:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2567
2025-11-11T17:18:25.998410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518413639892477:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21169855ns) {Configure:104(21167145ns) {ProcessConfigured:135(58923ns) {StartInitialHandshake:157(56697ns) {DropHandshakes:469(626ns) {DropIncomingHandshake:441(116ns) DropOutgoingHandshake:456(179ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(2208ns) {SwitchToState:178(730ns)}}}}}
2025-11-11T17:18:26.000952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518413639892480:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54078
2025-11-11T17:18:26.055725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518417934859788:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54078
2025-11-11T17:18:26.110995Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518417934859788:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.121740Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518413639892480:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.121918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518413639892484:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.121960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518413639892484:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.089701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518416520893553:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.113987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518412225926181:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.121887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518416520893554:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.129794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518412225926229:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.130310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893557:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.145796Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518413639892484:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:26.151690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518418904133773:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.151736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518418904133773:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.152759Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518418904133773:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:26.153114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518414609166473:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.153173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:26.153216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518414609166473:2075] [node 6] ICP22 created new session: [4:7571518418904133787:2075]
2025-11-11T17:18:26.153256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518418904133787:2075] [node 6] ICS09 handshake done sender: [4:7571518418904133773:2110] self: [4:7571518418904133772:0] peer: [6:7571518416520893556:0] socket: 103 qp: -1
2025-11-11T17:18:26.153293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518418904133787:2075] [node 6] ICS10 traffic start
2025-11-11T17:18:26.153355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:26.151309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518413616546943:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(73367354ns) {Configure:104(73357862ns) {ProcessConfigured:135(666388ns) {StartInitialHandshake:157(663755ns) {DropHandshakes:469(4167ns) {DropIncomingHandshake:441(948ns) DropOutgoingHandshake:456(1033ns)} GenerateSessionVirtualId:988(879ns) PrepareNewSessionHandshake:476(5737ns) {SwitchToState:178(1125ns)}}}}}
2025-11-11T17:18:26.178099Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518416520893553:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:26.157722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518412912014793:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(398985649ns) {Configure:104(398979387ns) {ProcessConfigured:135(35544356ns) {StartInitialHandshake:157(35540736ns) {DropHandshakes:469(3053ns) {DropIncomingHandshake:441(590ns) DropOutgoingHandshake:456(564ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:476(2752ns) {SwitchToState:178(1097ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.150711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467097:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.191506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518412225926181:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.191599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:26.191654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518412225926181:2050] [node 3] ICP22 created new session: [6:7571518416520893559:2050]
2025-11-11T17:18:26.191691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518416520893559:2050] [node 3] ICS09 handshake done sender: [6:7571518416520893553:2115] self: [6:7571518416520893555:0] peer: [3:7571518413639892483:0] socket: 105 qp: -1
2025-11-11T17:18:26.191728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518416520893559:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:26.191786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:26.192409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893558:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.206988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518412912014795:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(37114195ns) {Configure:104(37101915ns) {ProcessConfigured:135(63452ns) {StartInitialHandshake:157(61363ns) {DropHandshakes:469(3287ns) {DropIncomingHandshake:441(979ns) DropOutgoingHandshake:456(541ns)} GenerateSessionVirtualId:988(1245ns) PrepareNewSessionHandshake:476(4709ns) {SwitchToState:178(2090ns)}}}}}
2025-11-11T17:18:26.178278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518413639892477:2072] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.224679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518418904133790:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28224
2025-11-11T17:18:26.214110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518417206982156:2068] [node 4] ICP01 ready to work
2025-11-11T17:18:26.178369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.258146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518418904133791:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28224
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.267967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518416358467098:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2567
2025-11-11T17:18:26.268116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518416358467099:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2567
2025-11-11T17:18:26.223628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518417911514243:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8012
2025-11-11T17:18:26.214373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518417206982157:2076] [node 6] ICP01 ready to work
2025-11-11T17:18:26.178423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518413639892477:2072] [node 6] ICP22 created new session: [3:7571518417934859792:2072]
2025-11-11T17:18:26.282250Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467098:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.223792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518417911514244:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8012
2025-11-11T17:18:26.293641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518418904133785:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.223910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518417911514245:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8012
2025-11-11T17:18:26.178468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518417934859792:2072] [node 6] ICS09 handshake done sender: [3:7571518413639892484:2113] self: [3:7571518413639892483:0] peer: [6:7571518416520893555:0] socket: 102 qp: -1
2025-11-11T17:18:26.178505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518417934859792:2072] [node 6] ICS10 traffic start
2025-11-11T17:18:26.178561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:26.301683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518418904133790:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.309792Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518416520893554:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.293823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467099:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.317851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518412063499791:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.317872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518412063499791:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.224023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518417911514246:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8012
2025-11-11T17:18:26.309824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518414609166411:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.309913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518414609166412:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.325646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518418904133791:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.309979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982158:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.330429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518412225926229:2063] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.378178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518414609166413:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.378290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518418904133794:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28224
2025-11-11T17:18:26.378900Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518418904133790:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:26.381095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518414609166412:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.381135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:26.331885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518417934859795:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54078
2025-11-11T17:18:26.310106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982159:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-11-11T17:18:26.371468Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518417934859795:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.379041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518413639892486:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.354033Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518417206982158:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.379060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518413639892486:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.354187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982168:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-11-11T17:18:26.379918Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518413639892486:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:26.380123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518413639892478:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.380175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:26.380212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518413639892478:2074] [node 4] ICP22 created new session: [3:7571518417934859798:2074]
2025-11-11T17:18:26.380257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518417934859798:2074] [node 4] ICS09 handshake done sender: [3:7571518413639892486:2114] self: [3:7571518413639892485:0] peer: [4:7571518418904133795:0] socket: 128 qp: -1
2025-11-11T17:18:26.380284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518417934859798:2074] [node 4] ICS10 traffic start
2025-11-11T17:18:26.380319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:26.381178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518414609166412:2049] [node 2] ICP22 created new session: [4:7571518418904133797:2049]
2025-11-11T17:18:26.381232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518418904133797:2049] [node 2] ICS09 handshake done sender: [4:7571518418904133790:2116] self: [4:7571518418904133793:0] peer: [2:7571518417911514247:0] socket: 127 qp: -1
2025-11-11T17:18:26.381258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518418904133797:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:26.381295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.229954Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514245:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.330524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:26.251053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514244:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.285821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514246:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.313679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514243:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.313882Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514248:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.313906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518417911514248:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.314467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518413616546879:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.317791Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518417911514248:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.418349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518413616546944:2076] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.418422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:26.418469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518413616546944:2076] [node 4] ICP22 created new session: [2:7571518417911514265:2076]
2025-11-11T17:18:26.418509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518417911514265:2076] [node 4] ICS09 handshake done sender: [2:7571518417911514248:2115] self: [2:7571518417911514247:0] peer: [4:7571518418904133793:0] socket: 125 qp: -1
2025-11-11T17:18:26.418545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518417911514265:2076] [node 4] ICS10 traffic start
2025-11-11T17:18:26.418588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:26.382123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518412063499788:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.386745Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16083) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:26.401678Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518417206982159:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.330564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518412225926229:2063] [node 4] ICP22 created new session: [6:7571518416520893561:2063]
2025-11-11T17:18:26.330615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518416520893561:2063] [node 4] ICS09 handshake done sender: [6:7571518416520893554:2116] self: [6:7571518416520893556:0] peer: [4:7571518418904133772:0] socket: 110 qp: -1
2025-11-11T17:18:26.330649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518416520893561:2063] [node 4] ICS10 traffic start
2025-11-11T17:18:26.330677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:26.382155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518412063499788:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.403195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518416358467103:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2567
2025-11-11T17:18:26.403343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518416358467104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2567
2025-11-11T17:18:26.409717Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518412063499791:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:26.420096Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518412063499788:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:26.420377Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467104:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.420437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467103:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.420472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518412063499785:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.420514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:26.420545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518412063499785:2067] [node 4] ICP22 created new session: [1:7571518416358467107:2067]
2025-11-11T17:18:26.420588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518416358467107:2067] [node 4] ICS09 handshake done sender: [1:7571518412063499791:2111] self: [1:7571518412063499790:0] peer: [4:7571518418904133792:0] socket: 108 qp: -1
2025-11-11T17:18:26.420621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518416358467107:2067] [node 4] ICS10 traffic start
2025-11-11T17:18:26.420648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:26.420949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518412063499722:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.420986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:26.421018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518412063499722:2048] [node 2] ICP22 created new session: [1:7571518416358467109:2048]
2025-11-11T17:18:26.421056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518416358467109:2048] [node 2] ICS09 handshake done sender: [1:7571518412063499788:2110] self: [1:7571518412063499787:0] peer: [2:7571518417911514261:0] socket: 107 qp: -1
2025-11-11T17:18:26.421077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518416358467109:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:26.421136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.433853Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518417206982168:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.434049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518412912014793:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.387613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518418904133796:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28224
2025-11-11T17:18:26.434225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982171:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-11-11T17:18:26.434443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518412912014794:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.434488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982172:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.387918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518414609166413:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16083) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518418904133791:2117] held: no
2025-11-11T17:18:26.388075Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518418904133791:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:26.388239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518414609166413:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.388269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:26.388303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518414609166413:2050] [node 3] ICP22 created new session: [4:7571518418904133800:2050]
2025-11-11T17:18:26.388354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518418904133800:2050] [node 3] ICS09 handshake done sender: [4:7571518418904133791:2117] self: [4:7571518418904133795:0] peer: [3:7571518413639892485:0] socket: 129 qp: -1
2025-11-11T17:18:26.388388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518418904133800:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:26.388428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:26.406184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518418904133802:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28224
2025-11-11T17:18:26.422187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518418904133785:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:26.441780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518414609166411:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.441836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:26.441883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518414609166411:2048] [node 1] ICP22 created new session: [4:7571518418904133803:2048]
2025-11-11T17:18:26.441925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518418904133803:2048] [node 1] ICS09 handshake done sender: [4:7571518418904133785:2115] self: [4:7571518418904133792:0] peer: [1:7571518412063499790:0] socket: 113 qp: -1
2025-11-11T17:18:26.441953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518418904133803:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:26.441989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:26.442558Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:26.450962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518417911514263:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8012
2025-11-11T17:18:26.451097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518417911514264:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8012
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.455845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518417934859801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54078
2025-11-11T17:18:26.449755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467089:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.449793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518416358467089:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.454597Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518416358467089:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:26.454881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518412063499786:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.454939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:26.454981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518412063499786:2068] [node 5] ICP22 created new session: [1:7571518416358467116:2068]
2025-11-11T17:18:26.455024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518416358467116:2068] [node 5] ICS09 handshake done sender: [1:7571518416358467089:2112] self: [1:7571518412063499792:0] peer: [5:7571518417206982173:0] socket: 109 qp: -1
2025-11-11T17:18:26.455056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518416358467116:2068] [node 5] ICS10 traffic start
2025-11-11T17:18:26.455086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.473877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893565:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.474989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518414609166411:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:26.474307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-11-11T17:18:26.476634Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518417206982171:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.476740Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518417206982159:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:26.475608Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:26.477060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518412912014793:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.503251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518413639892488:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.503285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518413639892488:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.504925Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514242:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.477114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:26.504961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518417911514242:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.505568Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514263:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.505662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518413616546879:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518417911514246:2114] held: no
2025-11-11T17:18:26.505934Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518417911514246:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:26.521746Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518413639892488:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:26.521902Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467093:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.521936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518416358467093:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.525516Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518417934859801:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.477158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518412912014793:2048] [node 1] ICP22 created new session: [5:7571518417206982176:2048]
2025-11-11T17:18:26.477226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518417206982176:2048] [node 1] ICS09 handshake done sender: [5:7571518417206982159:2113] self: [5:7571518417206982173:0] peer: [1:7571518412063499792:0] socket: 123 qp: -1
2025-11-11T17:18:26.477267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518417206982176:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:26.477303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:26.485947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518417206982156:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.502098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518417206982172:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.510730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518416520893565:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.502515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518412912014795:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.510552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-11-11T17:18:26.510605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982182:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-11-11T17:18:26.525181Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518417206982172:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.502913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518418904133781:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.526224Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518416358467093:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:26.526204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518413639892479:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.510984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518412225926179:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.511041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893566:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.524743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893568:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.534171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518416520893565:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:26.502945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518418904133781:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.530606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518418904133805:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28224
2025-11-11T17:18:26.538044Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518418904133781:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:26.534383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:26.545692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518412225926179:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.545767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:26.545813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518412225926179:2048] [node 1] ICP22 created new session: [6:7571518416520893569:2048]
2025-11-11T17:18:26.545846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518416520893569:2048] [node 1] ICS09 handshake done sender: [6:7571518416520893565:2119] self: [6:7571518416520893567:0] peer: [1:7571518416358467092:0] socket: 130 qp: -1
2025-11-11T17:18:26.545877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518416520893569:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:26.545908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:26.523224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518413616546879:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.523283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:26.523331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518413616546879:2048] [node 1] ICP22 created new session: [2:7571518417911514269:2048]
2025-11-11T17:18:26.523374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518417911514269:2048] [node 1] ICS09 handshake done sender: [2:7571518417911514246:2114] self: [2:7571518417911514261:0] peer: [1:7571518412063499787:0] socket: 122 qp: -1
2025-11-11T17:18:26.523408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518417911514269:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:26.523452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:26.526273Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518417911514242:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:26.554392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518413616546943:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.554452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:26.554500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518413616546943:2074] [node 5] ICP22 created new session: [2:7571518417911514273:2074]
2025-11-11T17:18:26.554553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518417911514273:2074] [node 5] ICS09 handshake done sender: [2:7571518417911514242:2110] self: [2:7571518417911514241:0] peer: [5:7571518417206982174:0] socket: 124 qp: -1
2025-11-11T17:18:26.554583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518417911514273:2074] [node 5] ICS10 traffic start
2025-11-11T17:18:26.554610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.541983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518412912014795:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.541714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518413639892479:2080] [node 5] ICP22 created new session: [3:7571518417934859804:2080]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.577841Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518416520893566:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.578058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518412225926180:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.579487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893572:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.581791Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514252:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.581830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518417911514252:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.550302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518417934859804:2080] [node 5] ICS09 handshake done sender: [3:7571518413639892488:2115] self: [3:7571518413639892487:0] peer: [5:7571518417206982180:0] socket: 147 qp: -1
2025-11-11T17:18:26.585802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518414609166475:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.590936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:26.542043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:26.542088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518412912014795:2050] [node 3] ICP22 created new session: [5:7571518417206982183:2050]
2025-11-11T17:18:26.542131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518417206982183:2050] [node 3] ICS09 handshake done sender: [5:7571518417206982172:2119] self: [5:7571518417206982180:0] peer: [3:7571518413639892487:0] socket: 148 qp: -1
2025-11-11T17:18:26.542163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518417206982183:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:26.542197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:26.563236Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:26.564155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518412912014794:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518417206982168:2117] held: no
2025-11-11T17:18:26.564599Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518417206982168:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:26.564901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518412912014794:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.564967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:26.565011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518412912014794:2049] [node 2] ICP22 created new session: [5:7571518417206982186:2049]
2025-11-11T17:18:26.565069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518417206982186:2049] [node 2] ICS09 handshake done sender: [5:7571518417206982168:2117] self: [5:7571518417206982174:0] peer: [2:7571518417911514241:0] socket: 131 qp: -1
2025-11-11T17:18:26.565120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518417206982186:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:26.565162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:26.565550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518417206982185:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-11-11T17:18:26.579073Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518417206982171:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:26.597786Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518417911514252:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:26.606107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518413616546942:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.606184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:26.606226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518413616546942:2073] [node 6] ICP22 created new session: [2:7571518417911514276:2073]
2025-11-11T17:18:26.606271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518417911514276:2073] [node 6] ICS09 handshake done sender: [2:7571518417911514252:2117] self: [2:7571518417911514251:0] peer: [6:7571518416520893571:0] socket: 126 qp: -1
2025-11-11T17:18:26.606307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518417911514276:2073] [node 6] ICS10 traffic start
2025-11-11T17:18:26.606347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:26.595001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518414609166475:2082] [node 5] ICP22 created new session: [4:7571518418904133807:2082]
2025-11-11T17:18:26.595060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518418904133807:2082] [node 5] ICS09 handshake done sender: [4:7571518418904133781:2114] self: [4:7571518418904133780:0] peer: [5:7571518417206982178:0] socket: 104 qp: -1
2025-11-11T17:18:26.595093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518418904133807:2082] [node 5] ICS10 traffic start
2025-11-11T17:18:26.595131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:26.602008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518414609166475:2082] [node 5] ICP35 event processing took too much time PendingConnection:278(20025568ns) {HandleHandshakeStatus:329(20019914ns) {DropIncomingHandshake:441(1193ns) DropOutgoingHandshake:456(893ns) SwitchToState:178(1006ns) ProcessPendingSessionEvents:495(6553611ns) {ForwardSessionEventToSession:612(6494836ns) ForwardSessionEventToSession:612(7085ns) ForwardSessionEventToSession:612(2438ns) ForwardSessionEventToSession:612(5879ns) ForwardSessionEventToSession:612(4222ns) ForwardSessionEventToSession:612(2219ns) ForwardSessionEventToSession:612(7590ns) ForwardSessionEventToSession:612(2113ns) ForwardSessionEventToSession:612(5482ns) ForwardSessionEventToSession:612(2077ns)}}}
2025-11-11T17:18:26.602182Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518418904133805:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.602582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518414609166475:2082] [node 5] ICP18 handshake (actor [4:7571518418904133805:2121]) is too old
2025-11-11T17:18:26.550342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518417934859804:2080] [node 5] ICS10 traffic start
2025-11-11T17:18:26.590551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518412063499789:2086] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.590618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:26.590692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518412063499789:2086] [node 6] ICP22 created new session: [1:7571518416358467119:2086]
2025-11-11T17:18:26.590741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518416358467119:2086] [node 6] ICS09 handshake done sender: [1:7571518416358467093:2114] self: [1:7571518416358467092:0] peer: [6:7571518416520893567:0] socket: 121 qp: -1
2025-11-11T17:18:26.590777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518416358467119:2086] [node 6] ICS10 traffic start
2025-11-11T17:18:26.590807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:26.550395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:26.614054Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.620554Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518412225926180:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518416520893566:2120] held: no
2025-11-11T17:18:26.620700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893573:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.621523Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518416520893566:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:26.621753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518412225926180:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.621797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:26.621839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518412225926180:2049] [node 2] ICP22 created new session: [6:7571518416520893574:2049]
2025-11-11T17:18:26.621878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518416520893574:2049] [node 2] ICS09 handshake done sender: [6:7571518416520893566:2120] self: [6:7571518416520893571:0] peer: [2:7571518417911514251:0] socket: 154 qp: -1
2025-11-11T17:18:26.621905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518416520893574:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:26.621963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:26.622588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518413639892479:2080] [node 5] ICP35 event processing took too much time PendingConnection:278(47601094ns) {HandleHandshakeStatus:329(47598313ns) {DropIncomingHandshake:441(16952ns) DropOutgoingHandshake:456(514ns) SwitchToState:178(620ns) ProcessPendingSessionEvents:495(23310504ns) {ForwardSessionEventToSession:612(7565174ns) ForwardSessionEventToSession:612(4506ns) ForwardSessionEventToSession:612(2214ns) ForwardSessionEventToSession:612(7746713ns) ForwardSessionEventToSession:612(2069ns) ForwardSessionEventToSession:612(69202ns) ForwardSessionEventToSession:612(3484ns) ForwardSessionEventToSession:612(2601ns) ForwardSessionEventToSession:612(7894036ns) ForwardSessionEventToSession:612(3968ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.624359Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518416358467091:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.624390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518416358467091:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.623845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518417934859803:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54078
2025-11-11T17:18:26.624023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518413639892412:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.625750Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518416358467091:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.634035Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518417934859803:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.634289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518417934859807:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54078
2025-11-11T17:18:26.634764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518413639892413:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.634939Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518417934859801:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:26.635168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518413639892412:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.635220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:26.635260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518413639892412:2048] [node 1] ICP22 created new session: [3:7571518417934859809:2048]
2025-11-11T17:18:26.635300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518417934859809:2048] [node 1] ICS09 handshake done sender: [3:7571518417934859801:2118] self: [3:7571518417934859806:0] peer: [1:7571518416358467090:0] socket: 149 qp: -1
2025-11-11T17:18:26.635327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518417934859809:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:26.635362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:26.618789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518417206982156:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.618877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:26.618917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518417206982156:2068] [node 4] ICP22 created new session: [5:7571518417206982189:2068]
2025-11-11T17:18:26.618960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518417206982189:2068] [node 4] ICS09 handshake done sender: [5:7571518417206982171:2118] self: [5:7571518417206982178:0] peer: [4:7571518418904133780:0] socket: 144 qp: -1
2025-11-11T17:18:26.618997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518417206982189:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:26.619029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:26.619979Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:26.641989Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518417911514254:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.642031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518417911514254:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:26.645828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518412063499723:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.645930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:26.645987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518412063499723:2049] [node 3] ICP22 created new session: [1:7571518416358467122:2049]
2025-11-11T17:18:26.646029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518416358467122:2049] [node 3] ICS09 handshake done sender: [1:7571518416358467091:2113] self: [1:7571518416358467090:0] peer: [3:7571518417934859806:0] socket: 96 qp: -1
2025-11-11T17:18:26.646061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518416358467122:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:26.646100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:26.649819Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518417911514254:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:26.650100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518413616546880:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.650153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:26.650190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518413616546880:2049] [node 3] ICP22 created new session: [2:7571518417911514279:2049]
2025-11-11T17:18:26.650231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518417911514279:2049] [node 3] ICS09 handshake done sender: [2:7571518417911514254:2118] self: [2:7571518417911514253:0] peer: [3:7571518417934859808:0] socket: 156 qp: -1
2025-11-11T17:18:26.650263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518417911514279:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:26.650295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:26.669855Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:26.673901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518417206982156:2068] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.675677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518417934859811:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54078
2025-11-11T17:18:26.676117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518413639892413:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518417934859803:2119] held: no
2025-11-11T17:18:26.676272Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518417934859803:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:26.676472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518413639892413:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.676536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:26.676574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518413639892413:2049] [node 2] ICP22 created new session: [3:7571518417934859812:2049]
2025-11-11T17:18:26.676607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518417934859812:2049] [node 2] ICS09 handshake done sender: [3:7571518417934859803:2119] self: [3:7571518417934859808:0] peer: [2:7571518417911514253:0] socket: 158 qp: -1
2025-11-11T17:18:26.676643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518417934859812:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:26.676691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:26.685793Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518416520893572:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.686007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518412225926198:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:26.689791Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518417206982165:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:26.689832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518417206982165:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:26.691010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518416520893577:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58717
2025-11-11T17:18:26.693875Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518417206982165:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:26.694125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518417206982157:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:26.694183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:26.694225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518417206982157:2076] [node 6] ICP22 created new session: [5:7571518417206982192:2076]
2025-11-11T17:18:26.694266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518417206982192:2076] [node 6] ICS09 handshake done sender: [5:7571518417206982165:2116] self: [5:7571518417206982164:0] peer: [6:7571518416520893576:0] socket: 135 qp: -1
2025-11-11T17:18:26.694296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518417206982192:2076] [node 6] ICS10 traffic start
2025-11-11T17:18:26.694326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:26.698315Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30971) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:26.699390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518412225926198:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30971) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518416520893572:2122] held: no
2025-11-11T17:18:26.699506Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518416520893572:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:26.709791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518412225926198:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:26.709860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:26.709921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518412225926198:2059] [node 5] ICP22 created new session: [6:7571518416520893578:2059]
2025-11-11T17:18:26.709970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518416520893578:2059] [node 5] ICS09 handshake done sender: [6:7571518416520893572:2122] self: [6:7571518416520893576:0] peer: [5:7571518417206982164:0] socket: 138 qp: -1
2025-11-11T17:18:26.710008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518416520893578:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:26.710042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:26.755075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:26.755327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518417911514269:2048] [node 1] ICS01 socket: 122 reason# EndOfStream
2025-11-11T17:18:26.755363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518413616546879:2048] [node 1] ICP30 unregister session Session# [2:7571518417911514269:2048] VirtualId# [2:7571518417911514261:0]
2025-11-11T17:18:26.755394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518417911514269:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.755455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:26.755490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:26.762948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:26.763184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518417934859809:2048] [node 1] ICS01 socket: 149 reason# EndOfStream
2025-11-11T17:18:26.763220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518413639892412:2048] [node 1] ICP30 unregister session Session# [3:7571518417934859809:2048] VirtualId# [3:7571518417934859806:0]
2025-11-11T17:18:26.763249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518417934859809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.763308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:26.763344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:26.774302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:26.774480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518418904133803:2048] [node 1] ICS01 socket: 113 reason# EndOfStream
2025-11-11T17:18:26.774513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518414609166411:2048] [node 1] ICP30 unregister session Session# [4:7571518418904133803:2048] VirtualId# [4:7571518418904133792:0]
2025-11-11T17:18:26.774535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518418904133803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.774597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:26.774636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:26.773775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:26.774001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518417206982176:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-11T17:18:26.774037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518412912014793:2048] [node 1] ICP30 unregister session Session# [5:7571518417206982176:2048] VirtualId# [5:7571518417206982173:0]
2025-11-11T17:18:26.774078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518417206982176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.774161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:26.774210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:26.774715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:26.774887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518416520893569:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-11T17:18:26.774918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518412225926179:2048] [node 1] ICP30 unregister session Session# [6:7571518416520893569:2048] VirtualId# [6:7571518416520893567:0]
2025-11-11T17:18:26.774938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518416520893569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.774995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:26.775026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:26.823277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:26.823517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518417934859812:2049] [node 2] ICS01 socket: 158 reason# EndOfStream
2025-11-11T17:18:26.823555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518413639892413:2049] [node 2] ICP30 unregister session Session# [3:7571518417934859812:2049] VirtualId# [3:7571518417934859808:0]
2025-11-11T17:18:26.823584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518417934859812:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.823645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:26.823678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:26.845748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:26.845747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:26.845992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518418904133797:2049] [node 2] ICS01 socket: 127 reason# EndOfStream
2025-11-11T17:18:26.846033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518417206982186:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:18:26.846033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518414609166412:2049] [node 2] ICP30 unregister session Session# [4:7571518418904133797:2049] VirtualId# [4:7571518418904133793:0]
2025-11-11T17:18:26.846064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518418904133797:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.846080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518412912014794:2049] [node 2] ICP30 unregister session Session# [5:7571518417206982186:2049] VirtualId# [5:7571518417206982174:0]
2025-11-11T17:18:26.846109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518417206982186:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.846180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:26.846215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:26.845759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:26.845993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518416520893574:2049] [node 2] ICS01 socket: 154 reason# EndOfStream
2025-11-11T17:18:26.846140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:26.846044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518412225926180:2049] [node 2] ICP30 unregister session Session# [6:7571518416520893574:2049] VirtualId# [6:7571518416520893571:0]
2025-11-11T17:18:26.846064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518416520893574:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.846140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:26.846178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:26.846182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:26.961754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:26.961759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:26.961981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518417206982183:2050] [node 3] ICS01 socket: 148 reason# EndOfStream
2025-11-11T17:18:26.961980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518416520893559:2050] [node 3] ICS01 socket: 105 reason# EndOfStream
2025-11-11T17:18:26.962019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518412912014795:2050] [node 3] ICP30 unregister session Session# [5:7571518417206982183:2050] VirtualId# [5:7571518417206982180:0]
2025-11-11T17:18:26.962016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518412225926181:2050] [node 3] ICP30 unregister session Session# [6:7571518416520893559:2050] VirtualId# [6:7571518416520893555:0]
2025-11-11T17:18:26.962044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518417206982183:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.962045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518416520893559:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.962125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:26.962125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:26.962162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:26.962163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:26.969694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:26.969963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518418904133800:2050] [node 3] ICS01 socket: 129 reason# EndOfStream
2025-11-11T17:18:26.969999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518414609166413:2050] [node 3] ICP30 unregister session Session# [4:7571518418904133800:2050] VirtualId# [4:7571518418904133795:0]
2025-11-11T17:18:26.970032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518418904133800:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:26.970125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:26.970174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:27.021794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:27.022006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518416520893561:2063] [node 4] ICS01 socket: 110 reason# EndOfStream
2025-11-11T17:18:27.022048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518412225926229:2063] [node 4] ICP30 unregister session Session# [6:7571518416520893561:2063] VirtualId# [6:7571518416520893556:0]
2025-11-11T17:18:27.022079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518416520893561:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:27.022159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:27.022203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:27.022540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:27.022820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518417206982189:2068] [node 4] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-11T17:18:27.022857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518417206982189:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:27.022957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:27.023014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518417206982189:2068] [node 4] ICS15 start handshake
2025-11-11T17:18:27.029881Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16494) Couldn't connect to any resolved address
2025-11-11T17:18:27.033873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518417206982156:2068] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16494) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:27.033925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518417206982189:2068] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:27.033960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518417206982156:2068] [node 4] ICP30 unregister session Session# [5:7571518417206982189:2068] VirtualId# [5:7571518417206982178:0]
2025-11-11T17:18:27.034004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:27.034041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518417206982156:2068] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:27.034056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:27.045725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518417206982156:2068] [node 4] ICP33 wake up from error state
2025-11-11T17:18:27.117782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:27.118243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518416520893578:2059] [node 5] ICS01 socket: 138 reason# EndOfStream
2025-11-11T17:18:27.118336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518412225926198:2059] [node 5] ICP30 unregister session Session# [6:7571518416520893578:2059] VirtualId# [6:7571518416520893576:0]
2025-11-11T17:18:27.118426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518416520893578:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:27.118587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:27.118634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:27.205963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518420748851448:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:27.206004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518420748851449:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:27.237909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518421553931468:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:27.237963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518421553931469:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:27.293943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518422463980189:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:27.293986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518422463980190:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:27.353949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518421083742342:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:27.353995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518421083742343:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:27.354022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518421083742344:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:27.369433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518419437058122:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:27.369468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518419437058123:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:27.369490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518419437058124:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:27.425890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518422969803296:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:27.425936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518422969803297:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:27.425961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518422969803298:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:27.614695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518422969803311:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:27.617096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518421083742355:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
2025-11-11T17:18:27.585643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518422463980210:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:27.619045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518421553931495:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:27.633765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518422969803316:2059] [node 5] ICP01 ready to work
2025-11-11T17:18:27.718357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518422463980229:2066] [node 5] ICP01 ready to work
2025-11-11T17:18:27.718786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518422463980230:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:27.675193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518419437058135:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:27.699624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518421553931500:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:27.825963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518419437058166:2066] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:27.714198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518420748851488:2071] [node 4] ICP01 ready to work
2025-11-11T17:18:27.775378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518421083742385:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:27.860585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518422463980254:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-11-11T17:18:27.906444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518419437058166:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(80113806ns) {Configure:104(80107733ns) {ProcessConfigured:135(73801ns) {StartInitialHandshake:157(70628ns) {DropHandshakes:469(4034ns) {DropIncomingHandshake:441(944ns) DropOutgoingHandshake:456(763ns)} GenerateSessionVirtualId:988(987ns) PrepareNewSessionHandshake:476(2765ns) {SwitchToState:178(807ns)}}}}}
2025-11-11T17:18:27.777753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518420748851492:2073] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:27.946754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518421083742342:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(328918473ns) {Configure:104(328899977ns) {ProcessConfigured:135(75598ns) {StartInitialHandshake:157(73172ns) {DropHandshakes:469(3051ns) {DropIncomingHandshake:441(640ns) DropOutgoingHandshake:456(618ns)} GenerateSessionVirtualId:988(1076ns) PrepareNewSessionHandshake:476(2010ns) {SwitchToState:178(677ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:27.941636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518422969803298:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(260646374ns) {Configure:104(260639205ns) {ProcessConfigured:135(55200ns) {StartInitialHandshake:157(53544ns) {DropHandshakes:469(1607ns) {DropIncomingHandshake:441(370ns) DropOutgoingHandshake:456(269ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:476(1138ns) {SwitchToState:178(464ns)}}}}}
2025-11-11T17:18:27.994022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518422463980266:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-11-11T17:18:27.933299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518420748851522:2075] [node 6] ICP01 ready to work
2025-11-11T17:18:27.995226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518421083742414:2065] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.000307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518420748851513:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4917
2025-11-11T17:18:28.027377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518421083742408:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
2025-11-11T17:18:28.005630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518419437058189:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.034634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518421083742344:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(85083883ns) {Configure:104(85080470ns) {ProcessConfigured:135(97370ns) {StartInitialHandshake:157(95032ns) {DropHandshakes:469(3496ns) {DropIncomingHandshake:441(675ns) DropOutgoingHandshake:456(649ns)} GenerateSessionVirtualId:988(1079ns) PrepareNewSessionHandshake:476(1810ns) {SwitchToState:178(637ns)}}}}}
2025-11-11T17:18:28.041985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518420748851513:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.073855Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518422463980266:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.058282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518422969803296:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21207607ns) {Configure:104(21202345ns) {ProcessConfigured:135(72118ns) {StartInitialHandshake:157(69208ns) {DropHandshakes:469(3269ns) {DropIncomingHandshake:441(634ns) DropOutgoingHandshake:456(610ns)} GenerateSessionVirtualId:988(892ns) PrepareNewSessionHandshake:476(2078ns) {SwitchToState:178(627ns)}}}}}
2025-11-11T17:18:28.067572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518421553931469:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(473478673ns) {Configure:104(473473644ns) {ProcessConfigured:135(9844009ns) {StartInitialHandshake:157(9841690ns) {DropHandshakes:469(3231ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(578ns)} GenerateSessionVirtualId:988(1022ns) PrepareNewSessionHandshake:476(4380ns) {SwitchToState:178(2409ns)}}}}}
2025-11-11T17:18:28.042510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518425043818825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4917
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.077639Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518421083742408:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.078181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518422463980189:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.074344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518425848898835:2091] [node 5] ICP01 ready to work
2025-11-11T17:18:28.105650Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518420748851519:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.105818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770666:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.078258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518426758947564:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-11-11T17:18:28.078308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518426758947565:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-11-11T17:18:28.101918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518422463980232:2084] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.106020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518422463980232:2084] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.085882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518425378709711:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.105696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518420748851519:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.110132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518422969803316:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(408223498ns) {Configure:104(408214315ns) {ProcessConfigured:135(63172ns) {StartInitialHandshake:157(60504ns) {DropHandshakes:469(3117ns) {DropIncomingHandshake:441(576ns) DropOutgoingHandshake:456(733ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:476(2331ns) {SwitchToState:178(814ns)}}}}}
2025-11-11T17:18:28.119075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770671:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.119176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770672:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.130896Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518427264770671:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.149751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518422969803297:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.131157Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518422463980232:2084] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:28.086273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518421083742344:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.146037Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518419437058189:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.076203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518425848898836:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50768
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.158274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770674:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.126098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518420748851515:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.159022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025495:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.123055Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518425848898834:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.086346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518425378709719:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
2025-11-11T17:18:28.123075Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518425848898834:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.123773Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518425848898834:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.126119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518420748851515:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.124373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518421553931500:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.159829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518422463980190:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(472011098ns) {Configure:104(471997707ns) {ProcessConfigured:135(42046342ns) {StartInitialHandshake:157(42043425ns) {DropHandshakes:469(3212ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(601ns)} GenerateSessionVirtualId:988(1116ns) PrepareNewSessionHandshake:476(4026ns) {SwitchToState:178(1452ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.117792Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518425378709711:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.159315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518419437058124:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.159411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025497:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.177912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518419437058135:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(436179666ns) {Configure:104(412155953ns) {ProcessConfigured:135(8185364ns) {StartInitialHandshake:157(8182606ns) {DropHandshakes:469(2924ns) {DropIncomingHandshake:441(566ns) DropOutgoingHandshake:456(525ns)} GenerateSessionVirtualId:988(546ns) PrepareNewSessionHandshake:476(2898ns) {SwitchToState:178(704ns)}}}}}
2025-11-11T17:18:28.174379Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518427264770671:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:28.199440Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518423732025495:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.199563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518419437058122:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.235551Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518419437058184:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.235577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518419437058184:2104] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.236501Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518419437058184:2104] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:28.236806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518419437058166:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.236848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:28.236893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518419437058166:2066] [node 6] ICP22 created new session: [5:7571518423732025502:2066]
2025-11-11T17:18:28.236933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518423732025502:2066] [node 6] ICS09 handshake done sender: [5:7571518419437058184:2104] self: [5:7571518419437058183:0] peer: [6:7571518427264770678:0] socket: 133 qp: -1
2025-11-11T17:18:28.236964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518423732025502:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:28.236996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.124408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:28.202351Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518426758947565:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.225970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518422969803297:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.138903Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518420748851519:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:28.263006Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518423732025497:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.121766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518421083742355:2060] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.206018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518426758947564:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.226047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:28.124450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518421553931500:2070] [node 4] ICP22 created new session: [2:7571518425848898843:2070]
2025-11-11T17:18:28.143059Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518420748851515:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.263294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518419437058123:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.121914Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518425378709719:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.206141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518422463980254:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.226090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518422969803297:2049] [node 2] ICP22 created new session: [6:7571518427264770675:2049]
2025-11-11T17:18:28.155095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518420748851449:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.155154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:28.155190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518420748851449:2049] [node 3] ICP22 created new session: [1:7571518425043818830:2049]
2025-11-11T17:18:28.155234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518425043818830:2049] [node 3] ICS09 handshake done sender: [1:7571518420748851519:2113] self: [1:7571518420748851518:0] peer: [3:7571518426758947566:0] socket: 114 qp: -1
2025-11-11T17:18:28.124492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518425848898843:2070] [node 4] ICS09 handshake done sender: [2:7571518425848898834:2112] self: [2:7571518425848898833:0] peer: [4:7571518425378709724:0] socket: 127 qp: -1
2025-11-11T17:18:28.124520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518425848898843:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:28.124550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:28.153876Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518425848898832:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.153900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518425848898832:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.154734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518425848898846:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50768
2025-11-11T17:18:28.158467Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518425848898832:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:28.254013Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:28.254480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518421553931468:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(554511199ns) {Configure:104(540174743ns) {ProcessConfigured:135(60615962ns) {StartInitialHandshake:157(40654136ns) {DropHandshakes:469(6043ns) {DropIncomingHandshake:441(1286ns) DropOutgoingHandshake:456(705ns)} GenerateSessionVirtualId:988(3702ns) PrepareNewSessionHandshake:476(3194ns) {SwitchToState:178(702ns)}}}}}
2025-11-11T17:18:28.254635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518421553931495:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.254735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:28.254783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518421553931495:2069] [node 6] ICP22 created new session: [2:7571518425848898850:2069]
2025-11-11T17:18:28.254848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518425848898850:2069] [node 6] ICS09 handshake done sender: [2:7571518425848898832:2111] self: [2:7571518425848898831:0] peer: [6:7571518427264770673:0] socket: 126 qp: -1
2025-11-11T17:18:28.254883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518425848898850:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:28.254924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:28.256266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518425848898852:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50768
2025-11-11T17:18:28.273882Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518425848898846:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.274035Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518425848898838:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.274060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518425848898838:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.274890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518425848898830:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.274909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518425848898830:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.278114Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518425848898838:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:28.285964Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518425848898830:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:28.304684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518425848898852:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.304998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518425848898835:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.305051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:28.305092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518425848898835:2091] [node 5] ICP22 created new session: [2:7571518425848898858:2091]
2025-11-11T17:18:28.305142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518425848898858:2091] [node 5] ICS09 handshake done sender: [2:7571518425848898838:2114] self: [2:7571518425848898837:0] peer: [5:7571518423732025505:0] socket: 140 qp: -1
2025-11-11T17:18:28.305178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518425848898858:2091] [node 5] ICS10 traffic start
2025-11-11T17:18:28.305212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:28.306224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518421553931469:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.306258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:28.306317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518421553931469:2049] [node 3] ICP22 created new session: [2:7571518425848898860:2049]
2025-11-11T17:18:28.306366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518425848898860:2049] [node 3] ICS09 handshake done sender: [2:7571518425848898830:2110] self: [2:7571518425848898829:0] peer: [3:7571518426758947575:0] socket: 125 qp: -1
2025-11-11T17:18:28.306396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518425848898860:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:28.306448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:28.306877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518425848898857:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50768
2025-11-11T17:18:28.306957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518421553931468:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.315676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518425848898863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50768
2025-11-11T17:18:28.316365Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518425848898852:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:28.316578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518421553931468:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.316625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:28.316664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518421553931468:2048] [node 1] ICP22 created new session: [2:7571518425848898864:2048]
2025-11-11T17:18:28.316699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518425848898864:2048] [node 1] ICS09 handshake done sender: [2:7571518425848898852:2117] self: [2:7571518425848898862:0] peer: [1:7571518420748851520:0] socket: 102 qp: -1
2025-11-11T17:18:28.316731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518425848898864:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:28.316767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:28.287092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025507:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.287149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025508:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.287292Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:28.289460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025509:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.289523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025510:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.325829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518419437058166:2066] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:28.155263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518425043818830:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:28.155292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:28.177696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518425043818829:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4917
2025-11-11T17:18:28.198199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518420748851488:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.198244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:28.198285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518420748851488:2071] [node 4] ICP22 created new session: [1:7571518425043818833:2071]
2025-11-11T17:18:28.198331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518425043818833:2071] [node 4] ICS09 handshake done sender: [1:7571518420748851515:2111] self: [1:7571518420748851514:0] peer: [4:7571518425378709722:0] socket: 112 qp: -1
2025-11-11T17:18:28.198362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518425043818833:2071] [node 4] ICS10 traffic start
2025-11-11T17:18:28.198395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:28.199901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518420748851517:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.199920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518420748851517:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.219664Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518420748851517:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:28.221930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518420748851492:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.221966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:28.222010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518420748851492:2073] [node 5] ICP22 created new session: [1:7571518425043818837:2073]
2025-11-11T17:18:28.222045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518425043818837:2073] [node 5] ICS09 handshake done sender: [1:7571518420748851517:2112] self: [1:7571518420748851516:0] peer: [5:7571518423732025500:0] socket: 113 qp: -1
2025-11-11T17:18:28.222070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518425043818837:2073] [node 5] ICS10 traffic start
2025-11-11T17:18:28.222098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:28.241908Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518425043818829:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.282221Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518425043818825:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.295232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518420748851488:2071] [node 4] ICP18 handshake (actor [1:7571518425043818825:2117]) is too old
2025-11-11T17:18:28.295303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518425043818840:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4917
2025-11-11T17:18:28.295403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518425043818841:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4917
2025-11-11T17:18:28.122227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518421083742342:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.358091Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:28.358355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518421553931469:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:28.206256Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518422463980234:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.122314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518421083742343:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.122631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518425378709721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
2025-11-11T17:18:28.124087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518425378709725:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
2025-11-11T17:18:28.125477Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518425378709719:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:28.138689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518421083742343:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.138763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:28.138810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518421083742343:2049] [node 2] ICP22 created new session: [4:7571518425378709726:2049]
2025-11-11T17:18:28.165709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518425378709726:2049] [node 2] ICS09 handshake done sender: [4:7571518425378709719:2118] self: [4:7571518425378709724:0] peer: [2:7571518425848898833:0] socket: 128 qp: -1
2025-11-11T17:18:28.165755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518425378709726:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:28.173657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:28.238455Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518421083742408:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:28.238772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518421083742343:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(108835273ns) {HandleHandshakeStatus:329(107565181ns) {DropIncomingHandshake:441(1645ns) DropOutgoingHandshake:456(8357ns) SwitchToState:178(486ns) ProcessPendingSessionEvents:495(28747570ns) {ForwardSessionEventToSession:612(28697629ns) ForwardSessionEventToSession:612(7374ns) ForwardSessionEventToSession:612(5814ns) ForwardSessionEventToSession:612(2126ns) ForwardSessionEventToSession:612(9375ns) ForwardSessionEventToSession:612(2632ns) ForwardSessionEventToSession:612(2235ns) ForwardSessionEventToSession:612(3969ns) ForwardSessionEventToSession:612(1858ns) ForwardSessionEventToSession:612(3986ns)}}}
2025-11-11T17:18:28.238877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518425378709728:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
2025-11-11T17:18:28.270427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518421083742344:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.270485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:28.270529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518421083742344:2050] [node 3] ICP22 created new session: [4:7571518425378709730:2050]
2025-11-11T17:18:28.270599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518425378709730:2050] [node 3] ICS09 handshake done sender: [4:7571518421083742408:2111] self: [4:7571518425378709720:0] peer: [3:7571518422463980231:0] socket: 111 qp: -1
2025-11-11T17:18:28.270632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518425378709730:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:28.270668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:28.271585Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518425378709711:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:28.276062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518421083742342:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.276109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:28.276145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518421083742342:2048] [node 1] ICP22 created new session: [4:7571518425378709732:2048]
2025-11-11T17:18:28.276239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518425378709732:2048] [node 1] ICS09 handshake done sender: [4:7571518425378709711:2115] self: [4:7571518425378709722:0] peer: [1:7571518420748851514:0] socket: 116 qp: -1
2025-11-11T17:18:28.276277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518425378709732:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:28.276321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:28.303477Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518421083742410:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.303502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518421083742410:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.307963Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518421083742410:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:28.308193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518421083742385:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.308251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:28.308292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518421083742385:2059] [node 6] ICP22 created new session: [4:7571518425378709735:2059]
2025-11-11T17:18:28.308337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518425378709735:2059] [node 6] ICS09 handshake done sender: [4:7571518421083742410:2112] self: [4:7571518421083742409:0] peer: [6:7571518427264770684:0] socket: 118 qp: -1
2025-11-11T17:18:28.308363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518425378709735:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:28.308395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:28.226134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518427264770675:2049] [node 2] ICS09 handshake done sender: [6:7571518427264770671:2116] self: [6:7571518427264770673:0] peer: [2:7571518425848898831:0] socket: 131 qp: -1
2025-11-11T17:18:28.226163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518427264770675:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:28.226197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:28.233826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518427264770672:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.235242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518422969803316:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.264059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770680:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.264473Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518427264770672:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:28.264626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518422969803316:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.264669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:28.264708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518422969803316:2059] [node 5] ICP22 created new session: [6:7571518427264770681:2059]
2025-11-11T17:18:28.264746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518427264770681:2059] [node 5] ICS09 handshake done sender: [6:7571518427264770672:2117] self: [6:7571518427264770678:0] peer: [5:7571518419437058183:0] socket: 135 qp: -1
2025-11-11T17:18:28.264780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518427264770681:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:28.264815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:28.266271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.293652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518427264770666:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.294154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518422969803311:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.294237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770683:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.307541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770685:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.329691Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518427264770679:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.363453Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23097) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:28.371045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518422969803311:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23097) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518427264770666:2114] held: no
2025-11-11T17:18:28.371170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518422969803296:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.371237Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518427264770666:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:28.314091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518425043818841:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.206291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518422463980234:2085] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.206883Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:28.217793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518422463980210:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.217839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:28.217884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518422463980210:2064] [node 4] ICP22 created new session: [3:7571518426758947571:2064]
2025-11-11T17:18:28.217923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518426758947571:2064] [node 4] ICS09 handshake done sender: [3:7571518422463980232:2084] self: [3:7571518422463980231:0] peer: [4:7571518425378709720:0] socket: 106 qp: -1
2025-11-11T17:18:28.217959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518426758947571:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:28.217999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:28.218485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518422463980189:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518422463980266:2114] held: no
2025-11-11T17:18:28.219086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518426758947570:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-11-11T17:18:28.272425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518422463980190:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.273042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518422463980266:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:28.289127Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518422463980234:2085] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:28.295771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518422463980189:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.295814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:28.295871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518422463980189:2048] [node 1] ICP22 created new session: [3:7571518426758947578:2048]
2025-11-11T17:18:28.295910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518426758947578:2048] [node 1] ICS09 handshake done sender: [3:7571518422463980266:2114] self: [3:7571518426758947566:0] peer: [1:7571518420748851518:0] socket: 115 qp: -1
2025-11-11T17:18:28.295942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518426758947578:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:28.296030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:28.320431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518426758947580:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-11-11T17:18:28.330227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518422463980229:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.330290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:28.330356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518422463980229:2066] [node 5] ICP22 created new session: [3:7571518426758947582:2066]
2025-11-11T17:18:28.330399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518426758947582:2066] [node 5] ICS09 handshake done sender: [3:7571518422463980234:2085] self: [3:7571518422463980233:0] peer: [5:7571518423732025498:0] socket: 107 qp: -1
2025-11-11T17:18:28.330430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518426758947582:2066] [node 5] ICS10 traffic start
2025-11-11T17:18:28.330466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:28.330850Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518426758947565:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:28.354984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518422463980190:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.355051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:28.355102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518422463980190:2049] [node 2] ICP22 created new session: [3:7571518426758947584:2049]
2025-11-11T17:18:28.355147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518426758947584:2049] [node 2] ICS09 handshake done sender: [3:7571518426758947565:2117] self: [3:7571518426758947575:0] peer: [2:7571518425848898829:0] socket: 129 qp: -1
2025-11-11T17:18:28.355185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518426758947584:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:28.355218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:28.382160Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20678) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:28.383409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518419437058123:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20678) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518423732025497:2116] held: no
2025-11-11T17:18:28.383554Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:28.314226Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518420748851521:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.314244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518420748851521:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.384218Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518423732025497:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:28.315901Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518420748851521:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.337811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518420748851448:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.337890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:28.337939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518420748851448:2048] [node 2] ICP22 created new session: [1:7571518425043818844:2048]
2025-11-11T17:18:28.337983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518425043818844:2048] [node 2] ICS09 handshake done sender: [1:7571518420748851521:2114] self: [1:7571518420748851520:0] peer: [2:7571518425848898862:0] socket: 119 qp: -1
2025-11-11T17:18:28.338018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518425043818844:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:28.338084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:28.389788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518425043818840:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.389946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518420748851492:2073] [node 5] ICP18 handshake (actor [1:7571518425043818840:2119]) is too old
2025-11-11T17:18:28.397714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518420748851525:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.397741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518420748851525:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.400652Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518420748851525:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:28.400898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518420748851522:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.400938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:28.400973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518420748851522:2075] [node 6] ICP22 created new session: [1:7571518425043818847:2075]
2025-11-11T17:18:28.401014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518425043818847:2075] [node 6] ICS09 handshake done sender: [1:7571518420748851525:2116] self: [1:7571518420748851524:0] peer: [6:7571518427264770686:0] socket: 148 qp: -1
2025-11-11T17:18:28.401042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518425043818847:2075] [node 6] ICS10 traffic start
2025-11-11T17:18:28.401107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:28.393806Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518427264770683:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.393869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518422969803311:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.399251Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518419437058189:2107] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:28.393930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:28.393979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518422969803311:2061] [node 4] ICP22 created new session: [6:7571518427264770687:2061]
2025-11-11T17:18:28.393954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518422969803298:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.394591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518427264770687:2061] [node 4] ICS09 handshake done sender: [6:7571518427264770666:2114] self: [6:7571518427264770684:0] peer: [4:7571518421083742409:0] socket: 121 qp: -1
2025-11-11T17:18:28.404138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518427264770687:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:28.404179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:28.404329Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:28.413957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518422969803296:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518427264770679:2119] held: no
2025-11-11T17:18:28.414317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770690:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.418192Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518427264770679:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:28.418361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518422969803296:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.418407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:28.418457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518422969803296:2048] [node 1] ICP22 created new session: [6:7571518427264770691:2048]
2025-11-11T17:18:28.418496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518427264770691:2048] [node 1] ICS09 handshake done sender: [6:7571518427264770679:2119] self: [6:7571518427264770686:0] peer: [1:7571518420748851524:0] socket: 151 qp: -1
2025-11-11T17:18:28.418528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518427264770691:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:28.418557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:28.421848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518422463980238:2088] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.421896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518422463980238:2088] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.434016Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518422463980238:2088] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:28.399499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518419437058124:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518419437058189:2107] held: no
2025-11-11T17:18:28.399601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518419437058124:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.399662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:28.399714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518419437058124:2050] [node 3] ICP22 created new session: [5:7571518423732025511:2050]
2025-11-11T17:18:28.399748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518423732025511:2050] [node 3] ICS09 handshake done sender: [5:7571518419437058189:2107] self: [5:7571518423732025498:0] peer: [3:7571518422463980233:0] socket: 110 qp: -1
2025-11-11T17:18:28.399781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518423732025511:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:28.399812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:28.400258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518423732025509:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.400368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025506:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.402640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518419437058123:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.402702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:28.402745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518419437058123:2049] [node 2] ICP22 created new session: [5:7571518423732025513:2049]
2025-11-11T17:18:28.402784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518423732025513:2049] [node 2] ICS09 handshake done sender: [5:7571518423732025497:2116] self: [5:7571518423732025505:0] peer: [2:7571518425848898837:0] socket: 147 qp: -1
2025-11-11T17:18:28.402808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518423732025513:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:28.402838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:28.403203Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:28.424389Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518423732025495:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:28.429781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518419437058122:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518423732025495:2115] held: no
2025-11-11T17:18:28.429955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518419437058135:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:28.437738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518419437058122:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.437787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:28.437834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518419437058122:2048] [node 1] ICP22 created new session: [5:7571518423732025517:2048]
2025-11-11T17:18:28.437878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518423732025517:2048] [node 1] ICS09 handshake done sender: [5:7571518423732025495:2115] self: [5:7571518423732025500:0] peer: [1:7571518420748851516:0] socket: 130 qp: -1
2025-11-11T17:18:28.437910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518423732025517:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:28.437944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.456830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518427264770693:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46132
2025-11-11T17:18:28.449711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518425378709713:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.449753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518425378709713:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:28.454192Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518425378709713:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:28.455594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518422463980230:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.455663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:28.455732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518422463980230:2069] [node 6] ICP22 created new session: [3:7571518426758947587:2069]
2025-11-11T17:18:28.455776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518426758947587:2069] [node 6] ICS09 handshake done sender: [3:7571518422463980238:2088] self: [3:7571518422463980237:0] peer: [6:7571518427264770688:0] socket: 159 qp: -1
2025-11-11T17:18:28.455807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518426758947587:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:28.455849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:28.457663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518427264770683:2121] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:28.458046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518421083742414:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:28.458087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:28.458121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518421083742414:2065] [node 5] ICP22 created new session: [4:7571518425378709739:2065]
2025-11-11T17:18:28.458161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518425378709739:2065] [node 5] ICS09 handshake done sender: [4:7571518425378709713:2116] self: [4:7571518425378709712:0] peer: [5:7571518423732025516:0] socket: 124 qp: -1
2025-11-11T17:18:28.458188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518425378709739:2065] [node 5] ICS10 traffic start
2025-11-11T17:18:28.458211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:28.458582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518425378709738:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14682
2025-11-11T17:18:28.459521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518422969803298:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.459567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:28.459607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518422969803298:2050] [node 3] ICP22 created new session: [6:7571518427264770694:2050]
2025-11-11T17:18:28.459641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518427264770694:2050] [node 3] ICS09 handshake done sender: [6:7571518427264770683:2121] self: [6:7571518427264770688:0] peer: [3:7571518422463980237:0] socket: 117 qp: -1
2025-11-11T17:18:28.459669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518427264770694:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:28.459698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:28.461570Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518425378709738:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:28.455141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518423732025519:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21829
2025-11-11T17:18:28.459320Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518423732025509:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:28.461885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518419437058135:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:28.461950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:28.461992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518419437058135:2059] [node 4] ICP22 created new session: [5:7571518423732025520:2059]
2025-11-11T17:18:28.462021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518423732025520:2059] [node 4] ICS09 handshake done sender: [5:7571518423732025509:2121] self: [5:7571518423732025516:0] peer: [4:7571518425378709712:0] socket: 153 qp: -1
2025-11-11T17:18:28.462046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518423732025520:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:28.462078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:28.463857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518421083742414:2065] [node 5] ICP18 handshake (actor [4:7571518425378709738:2122]) is too old
2025-11-11T17:18:28.537771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:28.538019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518426758947578:2048] [node 1] ICS01 socket: 115 reason# EndOfStream
2025-11-11T17:18:28.538056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518422463980189:2048] [node 1] ICP30 unregister session Session# [3:7571518426758947578:2048] VirtualId# [3:7571518426758947566:0]
2025-11-11T17:18:28.538086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518426758947578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.538164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:28.538221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:28.541714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:28.541935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518425848898864:2048] [node 1] ICS01 socket: 102 reason# EndOfStream
2025-11-11T17:18:28.541974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518421553931468:2048] [node 1] ICP30 unregister session Session# [2:7571518425848898864:2048] VirtualId# [2:7571518425848898862:0]
2025-11-11T17:18:28.542002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518425848898864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.542282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:28.542338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:28.545794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:28.546261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:28.546448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518425378709732:2048] [node 1] ICS01 socket: 116 reason# EndOfStream
2025-11-11T17:18:28.546000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518423732025517:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-11T17:18:28.546486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518421083742342:2048] [node 1] ICP30 unregister session Session# [4:7571518425378709732:2048] VirtualId# [4:7571518425378709722:0]
2025-11-11T17:18:28.546030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518419437058122:2048] [node 1] ICP30 unregister session Session# [5:7571518423732025517:2048] VirtualId# [5:7571518423732025500:0]
2025-11-11T17:18:28.546512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518425378709732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.546058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518423732025517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.546588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:28.546131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:28.546630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:28.546172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:28.546712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:28.546901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518427264770691:2048] [node 1] ICS01 socket: 151 reason# EndOfStream
2025-11-11T17:18:28.546927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518422969803296:2048] [node 1] ICP30 unregister session Session# [6:7571518427264770691:2048] VirtualId# [6:7571518427264770686:0]
2025-11-11T17:18:28.546949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518427264770691:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.547003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:28.547034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:28.585769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:28.586009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518426758947584:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-11T17:18:28.586053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518422463980190:2049] [node 2] ICP30 unregister session Session# [3:7571518426758947584:2049] VirtualId# [3:7571518426758947575:0]
2025-11-11T17:18:28.586083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518426758947584:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.586187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:28.586236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:28.601777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:28.602002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518423732025513:2049] [node 2] ICS01 socket: 147 reason# EndOfStream
2025-11-11T17:18:28.602042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518419437058123:2049] [node 2] ICP30 unregister session Session# [5:7571518423732025513:2049] VirtualId# [5:7571518423732025505:0]
2025-11-11T17:18:28.602067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518423732025513:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.602195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:28.602273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:28.601776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:28.602005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518427264770675:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:18:28.602039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518422969803297:2049] [node 2] ICP30 unregister session Session# [6:7571518427264770675:2049] VirtualId# [6:7571518427264770673:0]
2025-11-11T17:18:28.602069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518427264770675:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.602187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:28.602256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:28.602380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:28.602596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518425378709726:2049] [node 2] ICS07 socket disconnect 128 reason# EndOfStream
2025-11-11T17:18:28.602627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518425378709726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.602699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:28.602758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518425378709726:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:28.620780Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20678) Couldn't connect to any resolved address
2025-11-11T17:18:28.621075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518421083742343:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20678) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:28.621127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518425378709726:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:28.621161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518421083742343:2049] [node 2] ICP30 unregister session Session# [4:7571518425378709726:2049] VirtualId# [4:7571518425378709724:0]
2025-11-11T17:18:28.621208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:28.621236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518421083742343:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:28.621250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:28.641864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518421083742343:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:28.689797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:28.690017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518425378709730:2050] [node 3] ICS01 socket: 111 reason# EndOfStream
2025-11-11T17:18:28.690050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518421083742344:2050] [node 3] ICP30 unregister session Session# [4:7571518425378709730:2050] VirtualId# [4:7571518425378709720:0]
2025-11-11T17:18:28.690080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518425378709730:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.690160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:28.690202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:28.690640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:28.690791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518427264770694:2050] [node 3] ICS01 socket: 117 reason# EndOfStream
2025-11-11T17:18:28.690813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518422969803298:2050] [node 3] ICP30 unregister session Session# [6:7571518427264770694:2050] VirtualId# [6:7571518427264770688:0]
2025-11-11T17:18:28.690837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518427264770694:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.690884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:28.690909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:28.690289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:28.690450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518423732025511:2050] [node 3] ICS01 socket: 110 reason# EndOfStream
2025-11-11T17:18:28.690475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518419437058124:2050] [node 3] ICP30 unregister session Session# [5:7571518423732025511:2050] VirtualId# [5:7571518423732025498:0]
2025-11-11T17:18:28.690496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518423732025511:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.690541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:28.690572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:28.733833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:28.734067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518427264770687:2061] [node 4] ICS01 socket: 121 reason# EndOfStream
2025-11-11T17:18:28.734103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518422969803311:2061] [node 4] ICP30 unregister session Session# [6:7571518427264770687:2061] VirtualId# [6:7571518427264770684:0]
2025-11-11T17:18:28.734133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518427264770687:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.734209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:28.734252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:28.734338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:28.734510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518423732025520:2059] [node 4] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:18:28.734532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518419437058135:2059] [node 4] ICP30 unregister session Session# [5:7571518423732025520:2059] VirtualId# [5:7571518423732025516:0]
2025-11-11T17:18:28.734552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518423732025520:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.734599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:28.734633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:28.793747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:28.793988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518427264770681:2059] [node 5] ICS01 socket: 135 reason# EndOfStream
2025-11-11T17:18:28.794028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518422969803316:2059] [node 5] ICP30 unregister session Session# [6:7571518427264770681:2059] VirtualId# [6:7571518427264770678:0]
2025-11-11T17:18:28.794056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518427264770681:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:28.794136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:28.794179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:28.989903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518425117848067:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:28.989957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518425117848068:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:29.073923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518428957797277:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:29.073968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518428957797278:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:29.137754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518431790830321:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:29.137806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518431790830322:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:29.186342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518429238113427:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:29.186390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518429238113428:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:29.186412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518429238113429:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:29.257860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518429077928510:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:29.257911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518429077928511:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:29.257936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518429077928512:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:29.472616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518429238113428:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(54665173ns) {Configure:104(54659555ns) {ProcessConfigured:135(74085ns) {StartInitialHandshake:157(71349ns) {DropHandshakes:469(3324ns) {DropIncomingHandshake:441(835ns) DropOutgoingHandshake:456(511ns)} GenerateSessionVirtualId:988(1147ns) PrepareNewSessionHandshake:476(2259ns) {SwitchToState:178(556ns)}}}}}
2025-11-11T17:18:29.329795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518430940859815:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:29.329838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518430940859816:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:29.329867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518430940859817:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:29.501683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518430940859835:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:29.521889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518429412815397:2071] [node 5] ICP01 ready to work
2025-11-11T17:18:29.569714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518429412815396:2072] [node 6] ICP01 ready to work
2025-11-11T17:18:29.517796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518428957797296:2062] [node 4] ICP01 ready to work
2025-11-11T17:18:29.613808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518430940859841:2063] [node 5] ICP01 ready to work
2025-11-11T17:18:29.657872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518429077928543:2063] [node 4] ICP01 ready to work
2025-11-11T17:18:29.629726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518431790830373:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:29.729649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518428957797343:2063] [node 5] ICP01 ready to work
2025-11-11T17:18:29.745862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518429238113481:2066] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.761022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518430940859867:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.807926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518429238113429:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(269883890ns) {Configure:104(269880050ns) {ProcessConfigured:135(57152ns) {StartInitialHandshake:157(55101ns) {DropHandshakes:469(1733ns) {DropIncomingHandshake:441(654ns) DropOutgoingHandshake:456(167ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:476(1498ns) {SwitchToState:178(426ns)}}}}}
2025-11-11T17:18:29.809719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518429238113495:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:29.812585Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518429238113495:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.738879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518431790830378:2071] [node 4] ICP01 ready to work
2025-11-11T17:18:29.681704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518429412815427:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:29.837746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518428957797346:2071] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.853528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518429238113427:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(428102674ns) {Configure:104(428098404ns) {ProcessConfigured:135(47622329ns) {StartInitialHandshake:157(47619515ns) {DropHandshakes:469(3990ns) {DropIncomingHandshake:441(627ns) DropOutgoingHandshake:456(1018ns)} GenerateSessionVirtualId:988(813ns) PrepareNewSessionHandshake:476(3641ns) {SwitchToState:178(2099ns)}}}}}
2025-11-11T17:18:29.854910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518429238113427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.853090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518428957797308:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11877
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.747162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518429412815429:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1589
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.833921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518431790830388:2076] [node 5] ICP01 ready to work
2025-11-11T17:18:29.865152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518429238113504:2069] [node 5] ICP01 ready to work
2025-11-11T17:18:29.855793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518429077928569:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:29.872599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518428957797309:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11877
2025-11-11T17:18:29.874762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518428957797277:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(340759227ns) {Configure:104(340755535ns) {ProcessConfigured:135(60778ns) {StartInitialHandshake:157(57930ns) {DropHandshakes:469(2897ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(765ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:476(1620ns) {SwitchToState:178(595ns)}}}}}
2025-11-11T17:18:29.874872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518428957797344:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11877
2025-11-11T17:18:29.875031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518428957797345:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11877
2025-11-11T17:18:29.875190Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797308:2077] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.877837Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797345:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.747355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518429412815430:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1589
2025-11-11T17:18:29.865677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815429:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.865805Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815430:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.894021Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815434:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.894062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518429412815434:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:29.906146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518429238113507:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:29.847381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518431790830389:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-11T17:18:29.876787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518430940859890:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:29.933752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518429412815434:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.876888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518430940859891:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:29.927099Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797309:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.847518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518431790830390:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-11T17:18:29.888820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518431790830391:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-11T17:18:29.889629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518431790830392:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-11T17:18:29.903082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518431790830389:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.903159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518431790830390:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.921942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518431790830392:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.934140Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518431790830391:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.927208Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797344:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.945890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815436:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.884878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518430940859841:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(226959965ns) {Configure:104(226953325ns) {ProcessConfigured:135(74336ns) {StartInitialHandshake:157(71808ns) {DropHandshakes:469(2637ns) {DropIncomingHandshake:441(500ns) DropOutgoingHandshake:456(564ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:476(1748ns) {SwitchToState:178(524ns)}}}}}
2025-11-11T17:18:29.897661Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518430940859867:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.930022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518430940859890:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.930358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518430940859815:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:29.945988Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518430940859891:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.949945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518430940859816:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:29.953864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518430940859894:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:29.975221Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518430940859894:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.981635Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:29.953752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518429238113507:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.950236Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797352:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.945930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518429412815436:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:29.970114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518429077928543:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(263982697ns) {Configure:104(263975976ns) {ProcessConfigured:135(67176ns) {StartInitialHandshake:157(64247ns) {DropHandshakes:469(2727ns) {DropIncomingHandshake:441(462ns) DropOutgoingHandshake:456(518ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:476(1115ns) {SwitchToState:178(331ns)}}}}}
2025-11-11T17:18:29.974188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518429412815427:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:29.974235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:29.974283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518429412815427:2070] [node 4] ICP22 created new session: [1:7571518429412815448:2070]
2025-11-11T17:18:29.974325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518429412815448:2070] [node 4] ICS09 handshake done sender: [1:7571518429412815434:2113] self: [1:7571518429412815433:0] peer: [4:7571518429238113503:0] socket: 122 qp: -1
2025-11-11T17:18:29.974361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518429412815448:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:29.974395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:29.977718Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518429412815436:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:29.978522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518429412815447:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1589
2025-11-11T17:18:29.994715Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815447:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.994778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518429412815396:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:29.994879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:29.994927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518429412815396:2072] [node 6] ICP22 created new session: [1:7571518429412815452:2072]
2025-11-11T17:18:29.994976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518429412815452:2072] [node 6] ICS09 handshake done sender: [1:7571518429412815436:2114] self: [1:7571518429412815435:0] peer: [6:7571518430940859895:0] socket: 123 qp: -1
2025-11-11T17:18:29.995008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518429412815452:2072] [node 6] ICS10 traffic start
2025-11-11T17:18:29.995049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:30.001864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518425117848067:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(347926249ns) {Configure:104(347917537ns) {ProcessConfigured:135(23942915ns) {StartInitialHandshake:157(23940059ns) {DropHandshakes:469(2157ns) {DropIncomingHandshake:441(285ns) DropOutgoingHandshake:456(695ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:476(3042ns) {SwitchToState:178(1180ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.973007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518429238113428:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:29.989895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518430940859835:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(472487456ns) {Configure:104(472482394ns) {ProcessConfigured:135(11909430ns) {StartInitialHandshake:157(11907296ns) {DropHandshakes:469(3062ns) {DropIncomingHandshake:441(708ns) DropOutgoingHandshake:456(601ns)} GenerateSessionVirtualId:988(775ns) PrepareNewSessionHandshake:476(3219ns) {SwitchToState:178(464ns)}}}}}
2025-11-11T17:18:29.950258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518428957797352:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:29.997799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518431790830321:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(471949938ns) {Configure:104(471944914ns) {ProcessConfigured:135(7929656ns) {StartInitialHandshake:157(7925851ns) {DropHandshakes:469(3417ns) {DropIncomingHandshake:441(813ns) DropOutgoingHandshake:456(690ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:476(2567ns) {SwitchToState:178(1060ns)}}}}}
2025-11-11T17:18:30.021850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518431790830321:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.990128Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518430940859816:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518430940859891:2115] held: no
2025-11-11T17:18:29.970427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518429077928584:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:29.974116Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518429077928584:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.974211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518429077928511:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.014588Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:30.050732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518431790830394:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.050761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518431790830394:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.951047Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518428957797352:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:30.006718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518433533080806:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:29.951348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518428957797346:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.009848Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:29.990684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518430940859835:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.053790Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518431790830394:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.951401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:30.069638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815432:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.069688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518429412815432:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.075925Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518429412815432:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:30.076207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518425117848068:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.076254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:30.076293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518425117848068:2049] [node 3] ICP22 created new session: [1:7571518433707782753:2049]
2025-11-11T17:18:30.076334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518433707782753:2049] [node 3] ICS09 handshake done sender: [1:7571518429412815432:2112] self: [1:7571518429412815431:0] peer: [3:7571518436085797703:0] socket: 121 qp: -1
2025-11-11T17:18:30.076361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518433707782753:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:30.076396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:30.066432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518429077928569:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(172275957ns) {Configure:104(172266152ns) {ProcessConfigured:135(4419161ns) {StartInitialHandshake:157(4416294ns) {DropHandshakes:469(3698ns) {DropIncomingHandshake:441(1077ns) DropOutgoingHandshake:456(625ns)} GenerateSessionVirtualId:988(859ns) PrepareNewSessionHandshake:476(2681ns) {SwitchToState:178(661ns)}}}}}
2025-11-11T17:18:30.066609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518429077928511:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518429077928584:2113] held: no
2025-11-11T17:18:30.066852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518429077928585:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.066950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518433372895886:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.074473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518433372895886:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.074838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518433372895889:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.074959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518429077928510:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.951462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518428957797346:2071] [node 6] ICP22 created new session: [2:7571518428957797362:2071]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.058099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518429238113428:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518429238113507:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.017765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518430940859898:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.114495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518436085797705:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-11T17:18:30.114612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518436085797706:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-11T17:18:29.951513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518428957797362:2071] [node 6] ICS09 handshake done sender: [2:7571518428957797352:2116] self: [2:7571518428957797351:0] peer: [6:7571518430940859896:0] socket: 135 qp: -1
2025-11-11T17:18:30.058391Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518429238113507:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:30.022143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518430940859817:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.105848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815450:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.951545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518428957797362:2071] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.113429Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518433372895889:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.022253Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:30.105882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518429412815450:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.121998Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518436085797705:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.122127Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518431790830396:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.122148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518431790830396:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.122799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518431790830322:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.123121Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518431790830396:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:30.123341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518431790830388:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.123396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:30.123445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518431790830388:2076] [node 5] ICP22 created new session: [3:7571518436085797709:2076]
2025-11-11T17:18:30.123487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518436085797709:2076] [node 5] ICS09 handshake done sender: [3:7571518431790830396:2116] self: [3:7571518431790830395:0] peer: [5:7571518433372895893:0] socket: 139 qp: -1
2025-11-11T17:18:30.123519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518436085797709:2076] [node 5] ICS10 traffic start
2025-11-11T17:18:30.123551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:30.133861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518431790830373:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.133920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:30.133979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518431790830373:2059] [node 6] ICP22 created new session: [3:7571518436085797712:2059]
2025-11-11T17:18:30.134018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518436085797712:2059] [node 6] ICS09 handshake done sender: [3:7571518431790830394:2115] self: [3:7571518431790830393:0] peer: [6:7571518435235827198:0] socket: 141 qp: -1
2025-11-11T17:18:30.134043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518436085797712:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:30.134087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:30.135295Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518431790830390:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:30.135962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518431790830321:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.136023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:30.136071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518431790830321:2048] [node 1] ICP22 created new session: [3:7571518436085797715:2048]
2025-11-11T17:18:30.136108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518436085797715:2048] [node 1] ICS09 handshake done sender: [3:7571518431790830390:2112] self: [3:7571518436085797703:0] peer: [1:7571518429412815431:0] socket: 130 qp: -1
2025-11-11T17:18:30.136137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518436085797715:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:30.136183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:30.170653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518431790830399:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.170691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518431790830399:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.171777Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518431790830399:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:30.172008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518431790830378:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.172058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:30.172099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518431790830378:2071] [node 4] ICP22 created new session: [3:7571518436085797718:2071]
2025-11-11T17:18:30.172140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518436085797718:2071] [node 4] ICS09 handshake done sender: [3:7571518431790830399:2117] self: [3:7571518431790830398:0] peer: [4:7571518433533080820:0] socket: 129 qp: -1
2025-11-11T17:18:30.172166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518436085797718:2071] [node 4] ICS10 traffic start
2025-11-11T17:18:30.172196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:30.061651Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518429238113494:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.050319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518430940859815:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518430940859867:2093] held: no
2025-11-11T17:18:30.117972Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518429077928585:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.951577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:30.061696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518429238113494:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.106583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518433707782755:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1589
2025-11-11T17:18:30.118150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518429077928512:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:29.974422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797348:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.050550Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518430940859891:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:30.205994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518436085797720:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-11T17:18:30.066151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518429238113508:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:30.141833Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:30.142884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518429077928512:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518429077928585:2114] held: no
2025-11-11T17:18:30.143191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518433372895894:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.143309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518433372895895:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.144213Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518429077928585:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:30.144903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518429077928512:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.144946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:30.144984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518429077928512:2050] [node 3] ICP22 created new session: [5:7571518433372895897:2050]
2025-11-11T17:18:30.145022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518433372895897:2050] [node 3] ICS09 handshake done sender: [5:7571518429077928585:2114] self: [5:7571518433372895893:0] peer: [3:7571518431790830395:0] socket: 143 qp: -1
2025-11-11T17:18:30.145049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518433372895897:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:30.145082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:30.181723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518433372895894:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.182759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518433372895885:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.185728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518429077928543:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.202018Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518429077928584:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:30.202192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518429077928511:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.202256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:30.202295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518429077928511:2049] [node 2] ICP22 created new session: [5:7571518433372895900:2049]
2025-11-11T17:18:30.202347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518433372895900:2049] [node 2] ICS09 handshake done sender: [5:7571518429077928584:2113] self: [5:7571518429077928588:0] peer: [2:7571518428957797347:0] socket: 140 qp: -1
2025-11-11T17:18:30.202381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518433372895900:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:30.202412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:30.106629Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:30.229715Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518436085797705:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:29.974447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518428957797348:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.082604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518429238113428:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.109636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518429412815441:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.053300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518435235827201:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:30.242002Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:29.985650Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797350:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:29.985697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518428957797350:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:29.985975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518428957797348:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:29.987305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518428957797343:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:29.987362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:29.987411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518428957797343:2063] [node 5] ICP22 created new session: [2:7571518428957797366:2063]
2025-11-11T17:18:29.987455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518428957797366:2063] [node 5] ICS09 handshake done sender: [2:7571518428957797348:2114] self: [2:7571518428957797347:0] peer: [5:7571518429077928588:0] socket: 133 qp: -1
2025-11-11T17:18:29.987487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518428957797366:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:29.987527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:29.996240Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518428957797350:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:29.996566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518428957797296:2062] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:29.996607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:29.996649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518428957797296:2062] [node 4] ICP22 created new session: [2:7571518428957797370:2062]
2025-11-11T17:18:29.996682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518428957797370:2062] [node 4] ICS09 handshake done sender: [2:7571518428957797350:2115] self: [2:7571518428957797349:0] peer: [4:7571518429238113509:0] socket: 134 qp: -1
2025-11-11T17:18:29.996705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518428957797370:2062] [node 4] ICS10 traffic start
2025-11-11T17:18:29.996727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:30.057630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518428957797369:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11877
2025-11-11T17:18:30.081758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518428957797278:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(271418116ns) {Configure:104(271410357ns) {ProcessConfigured:135(36590776ns) {StartInitialHandshake:157(36587072ns) {DropHandshakes:469(3473ns) {DropIncomingHandshake:441(620ns) DropOutgoingHandshake:456(837ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:476(3893ns) {SwitchToState:178(1753ns)}}}}}
2025-11-11T17:18:30.093657Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518428957797369:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.098070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518428957797277:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.157898Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518433252764669:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.157941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518433252764669:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.162128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518433252764673:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11877
2025-11-11T17:18:30.182346Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518428957797369:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:30.193864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518428957797277:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.193901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:30.193942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518428957797277:2048] [node 1] ICP22 created new session: [2:7571518433252764675:2048]
2025-11-11T17:18:30.193983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518433252764675:2048] [node 1] ICS09 handshake done sender: [2:7571518428957797369:2118] self: [2:7571518433252764671:0] peer: [1:7571518429412815446:0] socket: 155 qp: -1
2025-11-11T17:18:30.194008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518433252764675:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:30.194037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:30.209944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518433252764669:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:30.229910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518428957797278:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.229992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:30.230034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518428957797278:2049] [node 3] ICP22 created new session: [2:7571518433252764677:2049]
2025-11-11T17:18:30.230083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518433252764677:2049] [node 3] ICS09 handshake done sender: [2:7571518433252764669:2119] self: [2:7571518433252764668:0] peer: [3:7571518436085797708:0] socket: 158 qp: -1
2025-11-11T17:18:30.230884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518433252764677:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:30.237863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:30.259343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518431790830322:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518436085797705:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.063040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518430940859816:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.082649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:30.109678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518429412815441:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.063091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:30.063135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518430940859816:2049] [node 2] ICP22 created new session: [6:7571518435235827202:2049]
2025-11-11T17:18:30.063179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518435235827202:2049] [node 2] ICS09 handshake done sender: [6:7571518430940859891:2115] self: [6:7571518430940859896:0] peer: [2:7571518428957797351:0] socket: 137 qp: -1
2025-11-11T17:18:30.063214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518435235827202:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:30.063245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:30.063982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518430940859897:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:30.070451Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518430940859867:2093] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:30.070655Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518430940859894:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:30.077690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518430940859815:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.093657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:30.094124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518430940859817:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.094192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:30.094243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518430940859817:2050] [node 3] ICP22 created new session: [6:7571518435235827205:2050]
2025-11-11T17:18:30.094284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518435235827205:2050] [node 3] ICS09 handshake done sender: [6:7571518430940859894:2117] self: [6:7571518435235827198:0] peer: [3:7571518431790830393:0] socket: 144 qp: -1
2025-11-11T17:18:30.094322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518435235827205:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:30.094381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:30.109679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518430940859815:2048] [node 1] ICP22 created new session: [6:7571518435235827204:2048]
2025-11-11T17:18:30.133679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518435235827204:2048] [node 1] ICS09 handshake done sender: [6:7571518430940859867:2093] self: [6:7571518430940859895:0] peer: [1:7571518429412815435:0] socket: 124 qp: -1
2025-11-11T17:18:30.141660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518435235827204:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:30.153681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:30.189939Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:30.191932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518430940859835:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518430940859890:2114] held: no
2025-11-11T17:18:30.192119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518435235827208:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:30.206431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518430940859815:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(96029400ns) {HandleHandshakeStatus:329(96027349ns) {DropIncomingHandshake:441(478ns) DropOutgoingHandshake:456(532ns) SwitchToState:178(289ns) ProcessPendingSessionEvents:495(1674ns)}}
2025-11-11T17:18:30.210045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518435235827209:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:30.210859Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518430940859890:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:30.225655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518430940859835:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.225730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:30.229841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518430940859835:2059] [node 4] ICP22 created new session: [6:7571518435235827210:2059]
2025-11-11T17:18:30.243022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518435235827210:2059] [node 4] ICS09 handshake done sender: [6:7571518430940859890:2114] self: [6:7571518430940859901:0] peer: [4:7571518429238113493:0] socket: 136 qp: -1
2025-11-11T17:18:30.243118Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518435235827209:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.253841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518435235827210:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:30.261893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:30.266427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518430940859841:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.262199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518431790830322:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.149874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518429412815450:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:30.270285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518433372895903:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.279009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518428957797278:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(52847899ns) {HandleHandshakeStatus:329(52845036ns) {DropIncomingHandshake:441(32451ns) DropOutgoingHandshake:456(601ns) SwitchToState:178(685ns) ProcessPendingSessionEvents:495(5072345ns) {ForwardSessionEventToSession:612(5025280ns) ForwardSessionEventToSession:612(8727ns) ForwardSessionEventToSession:612(4823ns) ForwardSessionEventToSession:612(2368ns) ForwardSessionEventToSession:612(4887ns) ForwardSessionEventToSession:612(2279ns) ForwardSessionEventToSession:612(2453ns) ForwardSessionEventToSession:612(4884ns) ForwardSessionEventToSession:612(2203ns) ForwardSessionEventToSession:612(4406ns)}}}
2025-11-11T17:18:30.082690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518429238113428:2049] [node 2] ICP22 created new session: [4:7571518433533080810:2049]
2025-11-11T17:18:30.285629Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518433372895888:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.285673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518433372895888:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.293763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518430940859835:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(67595784ns) {HandleHandshakeStatus:329(67593409ns) {DropIncomingHandshake:441(2546ns) DropOutgoingHandshake:456(663ns) SwitchToState:178(1202ns) ProcessPendingSessionEvents:495(1130ns)}}
2025-11-11T17:18:30.298562Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:30.299659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518429077928510:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518433372895886:2117] held: no
2025-11-11T17:18:30.299763Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518433372895894:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.300028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518433372895902:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44627
2025-11-11T17:18:30.300523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518429077928543:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.274719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:30.192437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518425117848067:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:30.082738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518433533080810:2049] [node 2] ICS09 handshake done sender: [4:7571518429238113507:2116] self: [4:7571518429238113509:0] peer: [2:7571518428957797349:0] socket: 138 qp: -1
2025-11-11T17:18:30.314090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518435235827213:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42811
2025-11-11T17:18:30.300567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:30.192491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:30.192532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518425117848067:2048] [node 2] ICP22 created new session: [1:7571518433707782758:2048]
2025-11-11T17:18:30.192575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518433707782758:2048] [node 2] ICS09 handshake done sender: [1:7571518429412815450:2117] self: [1:7571518429412815446:0] peer: [2:7571518433252764671:0] socket: 154 qp: -1
2025-11-11T17:18:30.192610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518433707782758:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:30.192640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:30.218050Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518429412815441:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:30.222695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518433707782757:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1589
2025-11-11T17:18:30.229688Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:30.245810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518429412815397:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.269704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:30.277790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518429412815397:2071] [node 5] ICP22 created new session: [1:7571518433707782761:2071]
2025-11-11T17:18:30.277847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518433707782761:2071] [node 5] ICS09 handshake done sender: [1:7571518429412815441:2115] self: [1:7571518429412815440:0] peer: [5:7571518433372895891:0] socket: 153 qp: -1
2025-11-11T17:18:30.277884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518433707782761:2071] [node 5] ICS10 traffic start
2025-11-11T17:18:30.277928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:30.278346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518429412815397:2071] [node 5] ICP35 event processing took too much time PendingConnection:278(32355810ns) {HandleHandshakeStatus:329(32352608ns) {DropIncomingHandshake:441(15020ns) DropOutgoingHandshake:456(426ns) SwitchToState:178(628ns) ProcessPendingSessionEvents:495(93632ns) {ForwardSessionEventToSession:612(46218ns) ForwardSessionEventToSession:612(5930ns) ForwardSessionEventToSession:612(2814ns) ForwardSessionEventToSession:612(1985ns) ForwardSessionEventToSession:612(4983ns) ForwardSessionEventToSession:612(1750ns) ForwardSessionEventToSession:612(2184ns) ForwardSessionEventToSession:612(11948ns) ForwardSessionEventToSession:612(2302ns) ForwardSessionEventToSession:612(5378ns)}}}
2025-11-11T17:18:30.279332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518431790830322:2049] [node 2] ICP22 created new session: [3:7571518436085797721:2049]
2025-11-11T17:18:30.082771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518433533080810:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:30.082806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:30.083607Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518429238113495:2109] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:30.095772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518429238113427:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.095854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:30.095895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518429238113427:2048] [node 1] ICP22 created new session: [4:7571518433533080813:2048]
2025-11-11T17:18:30.095932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518433533080813:2048] [node 1] ICS09 handshake done sender: [4:7571518429238113495:2109] self: [4:7571518429238113503:0] peer: [1:7571518429412815433:0] socket: 126 qp: -1
2025-11-11T17:18:30.095964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518433533080813:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:30.095999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:30.097043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518433533080812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:30.097292Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518429238113494:2108] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:30.098512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518433533080807:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:30.102490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518429238113481:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.102550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:30.102596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518429238113481:2066] [node 6] ICP22 created new session: [4:7571518433533080815:2066]
2025-11-11T17:18:30.102644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518433533080815:2066] [node 6] ICS09 handshake done sender: [4:7571518429238113494:2108] self: [4:7571518429238113493:0] peer: [6:7571518430940859901:0] socket: 128 qp: -1
2025-11-11T17:18:30.102678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518433533080815:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:30.102709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:30.121753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518433533080807:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.141790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518429238113481:2066] [node 6] ICP18 handshake (actor [4:7571518433533080807:2119]) is too old
2025-11-11T17:18:30.141932Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518433533080812:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.143822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518433533080818:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:30.170265Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518433533080818:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.170370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518429238113429:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:30.171612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518433533080821:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57915
2025-11-11T17:18:30.173157Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518433533080818:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:30.173322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518429238113429:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.173355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:30.173390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518429238113429:2050] [node 3] ICP22 created new session: [4:7571518433533080822:2050]
2025-11-11T17:18:30.173429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518433533080822:2050] [node 3] ICS09 handshake done sender: [4:7571518433533080818:2121] self: [4:7571518433533080820:0] peer: [3:7571518431790830398:0] socket: 169 qp: -1
2025-11-11T17:18:30.173474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518433533080822:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:30.173497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:30.217821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518429238113506:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:30.217859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518429238113506:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:30.225732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518429238113429:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(20001052ns) {HandleHandshakeStatus:329(19996213ns) {DropIncomingHandshake:441(298ns) DropOutgoingHandshake:456(3553ns) SwitchToState:178(122ns) ProcessPendingSessionEvents:495(19735639ns) {ForwardSessionEventToSession:612(343496ns) ForwardSessionEventToSession:612(3358ns) ForwardSessionEventToSession:612(20179ns) ForwardSessionEventToSession:612(3844602ns) ForwardSessionEventToSession:612(2686ns) ForwardSessionEventToSession:612(2736ns) ForwardSessionEventToSession:612(6801769ns) ForwardSessionEventToSession:612(6110ns) ForwardSessionEventToSession:612(4976ns) ForwardSessionEventToSession:612(2164ns)}}}
2025-11-11T17:18:30.241890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518429238113506:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:30.242163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518429238113504:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.242218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:30.242262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518429238113504:2069] [node 5] ICP22 created new session: [4:7571518433533080825:2069]
2025-11-11T17:18:30.266347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518433533080825:2069] [node 5] ICS09 handshake done sender: [4:7571518429238113506:2115] self: [4:7571518429238113505:0] peer: [5:7571518433372895899:0] socket: 163 qp: -1
2025-11-11T17:18:30.273874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518433533080825:2069] [node 5] ICS10 traffic start
2025-11-11T17:18:30.273956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:30.274322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518429238113504:2069] [node 5] ICP35 event processing took too much time PendingConnection:278(31987206ns) {HandleHandshakeStatus:329(31983614ns) {DropIncomingHandshake:441(13090ns) DropOutgoingHandshake:456(629ns) SwitchToState:178(549ns) ProcessPendingSessionEvents:495(84244ns) {ForwardSessionEventToSession:612(41190ns) ForwardSessionEventToSession:612(2947ns) ForwardSessionEventToSession:612(13145ns) ForwardSessionEventToSession:612(2376ns) ForwardSessionEventToSession:612(2255ns) ForwardSessionEventToSession:612(4020ns) ForwardSessionEventToSession:612(1538ns) ForwardSessionEventToSession:612(3906ns) ForwardSessionEventToSession:612(1735ns) ForwardSessionEventToSession:612(3269ns)}}}
2025-11-11T17:18:30.300605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518429077928543:2063] [node 4] ICP22 created new session: [5:7571518433372895905:2063]
2025-11-11T17:18:30.293975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518436085797721:2049] [node 2] ICS09 handshake done sender: [3:7571518436085797705:2119] self: [3:7571518436085797708:0] peer: [2:7571518433252764668:0] socket: 166 qp: -1
2025-11-11T17:18:30.305673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518436085797721:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:30.310191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:30.300646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518433372895905:2063] [node 4] ICS09 handshake done sender: [5:7571518433372895894:2120] self: [5:7571518433372895899:0] peer: [4:7571518429238113505:0] socket: 165 qp: -1
2025-11-11T17:18:30.300673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518433372895905:2063] [node 4] ICS10 traffic start
2025-11-11T17:18:30.300702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:30.301266Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518433372895886:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:30.309642Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:30.309997Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518433372895888:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:30.318144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518429077928543:2063] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:30.318214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518429077928510:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.318277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:30.318330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518429077928510:2048] [node 1] ICP22 created new session: [5:7571518433372895907:2048]
2025-11-11T17:18:30.318375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518433372895907:2048] [node 1] ICS09 handshake done sender: [5:7571518433372895886:2117] self: [5:7571518433372895891:0] peer: [1:7571518429412815440:0] socket: 114 qp: -1
2025-11-11T17:18:30.318407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518433372895907:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:30.318440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:30.318746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518429077928569:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:30.318803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:30.318840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518429077928569:2066] [node 6] ICP22 created new session: [5:7571518433372895909:2066]
2025-11-11T17:18:30.318871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518433372895909:2066] [node 6] ICS09 handshake done sender: [5:7571518433372895888:2118] self: [5:7571518433372895887:0] peer: [6:7571518435235827212:0] socket: 117 qp: -1
2025-11-11T17:18:30.318902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518433372895909:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:30.318926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:30.349720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518431790830322:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(68126326ns) {HandleHandshakeStatus:329(68123535ns) {DropIncomingHandshake:441(1064ns) DropOutgoingHandshake:456(527ns) SwitchToState:178(828ns) ProcessPendingSessionEvents:495(964ns)}}
2025-11-11T17:18:30.338138Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21422) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:30.350004Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518435235827209:2123] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:30.353801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518430940859841:2063] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21422) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518435235827209:2123] held: no
2025-11-11T17:18:30.381774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518430940859841:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:30.381847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:30.381911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518430940859841:2063] [node 5] ICP22 created new session: [6:7571518435235827214:2063]
2025-11-11T17:18:30.381983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518435235827214:2063] [node 5] ICS09 handshake done sender: [6:7571518435235827209:2123] self: [6:7571518435235827212:0] peer: [5:7571518433372895887:0] socket: 131 qp: -1
2025-11-11T17:18:30.382030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518435235827214:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:30.382070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:30.445641Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:30.445873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518433252764675:2048] [node 1] ICS01 socket: 155 reason# EndOfStream
2025-11-11T17:18:30.445906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518428957797277:2048] [node 1] ICP30 unregister session Session# [2:7571518433252764675:2048] VirtualId# [2:7571518433252764671:0]
2025-11-11T17:18:30.445941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518433252764675:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.446015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:30.446054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:30.450928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:30.451164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518436085797715:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-11T17:18:30.451200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518431790830321:2048] [node 1] ICP30 unregister session Session# [3:7571518436085797715:2048] VirtualId# [3:7571518436085797703:0]
2025-11-11T17:18:30.451234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518436085797715:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.451297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:30.451336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:30.461809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:30.462293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:30.462482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518435235827204:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-11-11T17:18:30.462510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518430940859815:2048] [node 1] ICP30 unregister session Session# [6:7571518435235827204:2048] VirtualId# [6:7571518430940859895:0]
2025-11-11T17:18:30.462534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518435235827204:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.462572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:30.462598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:30.464330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:30.464640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518433533080813:2048] [node 1] ICS07 socket disconnect 126 reason# EndOfStream
2025-11-11T17:18:30.464693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518433533080813:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.464819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:30.464888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518433533080813:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:30.462014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518433372895907:2048] [node 1] ICS01 socket: 114 reason# EndOfStream
2025-11-11T17:18:30.462051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518429077928510:2048] [node 1] ICP30 unregister session Session# [5:7571518433372895907:2048] VirtualId# [5:7571518433372895891:0]
2025-11-11T17:18:30.462081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518433372895907:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.462155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:30.462206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:30.478107Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Couldn't connect to any resolved address
2025-11-11T17:18:30.493838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518429238113427:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:30.493910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518433533080813:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:30.493948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518429238113427:2048] [node 1] ICP30 unregister session Session# [4:7571518433533080813:2048] VirtualId# [4:7571518429238113503:0]
2025-11-11T17:18:30.494007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:30.494046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518429238113427:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:30.494060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13574) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:30.505725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518429238113427:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:30.527282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:30.527515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518436085797721:2049] [node 2] ICS01 socket: 166 reason# EndOfStream
2025-11-11T17:18:30.527553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518431790830322:2049] [node 2] ICP30 unregister session Session# [3:7571518436085797721:2049] VirtualId# [3:7571518436085797708:0]
2025-11-11T17:18:30.527591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518436085797721:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.527653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:30.527686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:30.534993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:30.535241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518433533080810:2049] [node 2] ICS01 socket: 138 reason# EndOfStream
2025-11-11T17:18:30.535281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518429238113428:2049] [node 2] ICP30 unregister session Session# [4:7571518433533080810:2049] VirtualId# [4:7571518429238113509:0]
2025-11-11T17:18:30.535314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518433533080810:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.535406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:30.535446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:30.535869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:30.536064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518433372895900:2049] [node 2] ICS01 socket: 140 reason# EndOfStream
2025-11-11T17:18:30.536097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518429077928511:2049] [node 2] ICP30 unregister session Session# [5:7571518433372895900:2049] VirtualId# [5:7571518429077928588:0]
2025-11-11T17:18:30.536117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518433372895900:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.536160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:30.536186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:30.545813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:30.546052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518435235827202:2049] [node 2] ICS01 socket: 137 reason# EndOfStream
2025-11-11T17:18:30.546088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518430940859816:2049] [node 2] ICP30 unregister session Session# [6:7571518435235827202:2049] VirtualId# [6:7571518430940859896:0]
2025-11-11T17:18:30.546122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518435235827202:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.546240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:30.546286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:30.631781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:30.632041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518435235827205:2050] [node 3] ICS01 socket: 144 reason# EndOfStream
2025-11-11T17:18:30.632073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518430940859817:2050] [node 3] ICP30 unregister session Session# [6:7571518435235827205:2050] VirtualId# [6:7571518435235827198:0]
2025-11-11T17:18:30.632102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518435235827205:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.632197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:30.632241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:30.637674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:30.637896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518433372895897:2050] [node 3] ICS01 socket: 143 reason# EndOfStream
2025-11-11T17:18:30.637933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518429077928512:2050] [node 3] ICP30 unregister session Session# [5:7571518433372895897:2050] VirtualId# [5:7571518433372895893:0]
2025-11-11T17:18:30.637962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518433372895897:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.638031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:30.638064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:30.635757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:30.642378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518433533080822:2050] [node 3] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-11T17:18:30.642424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518433533080822:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.642524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:30.642565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518433533080822:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:30.652675Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Couldn't connect to any resolved address
2025-11-11T17:18:30.652808Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518429238113429:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:30.652861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518433533080822:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:30.652974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518429238113429:2050] [node 3] ICP30 unregister session Session# [4:7571518433533080822:2050] VirtualId# [4:7571518433533080820:0]
2025-11-11T17:18:30.653033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:30.653070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518429238113429:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:30.653084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:30.719495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:30.719725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518435235827210:2059] [node 4] ICS01 socket: 136 reason# EndOfStream
2025-11-11T17:18:30.719770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518430940859835:2059] [node 4] ICP30 unregister session Session# [6:7571518435235827210:2059] VirtualId# [6:7571518430940859901:0]
2025-11-11T17:18:30.719802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518435235827210:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.719909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:30.719950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:30.721724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:30.721905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518433372895905:2063] [node 4] ICS01 socket: 165 reason# EndOfStream
2025-11-11T17:18:30.721936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518429077928543:2063] [node 4] ICP30 unregister session Session# [5:7571518433372895905:2063] VirtualId# [5:7571518433372895899:0]
2025-11-11T17:18:30.721959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518433372895905:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.722036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:30.722076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:30.829759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:30.830008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518435235827214:2063] [node 5] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:18:30.830045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518430940859841:2063] [node 5] ICP30 unregister session Session# [6:7571518435235827214:2063] VirtualId# [6:7571518435235827212:0]
2025-11-11T17:18:30.830076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518435235827214:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:30.830177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:30.830221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:31.045882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518438953229500:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:31.045920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518438953229501:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:31.134793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518440546695379:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:31.134835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518440546695380:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:31.227378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518439613907167:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:31.227431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518439613907168:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:31.353910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518438388784330:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:31.353966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518438388784331:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:31.361633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518438388784329:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:31.405921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518439166240796:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:31.405962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518439166240797:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:31.405982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518439166240798:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:31.549871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518438424493573:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:31.549923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518438424493574:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:31.620151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518438424493572:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:31.674202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518439613907192:2067] [node 4] ICP01 ready to work
2025-11-11T17:18:31.679205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518438424493594:2067] [node 4] ICP01 ready to work
2025-11-11T17:18:31.725698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518438953229539:2065] [node 4] ICP01 ready to work
2025-11-11T17:18:31.713631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518439166240811:2060] [node 4] ICP01 ready to work
2025-11-11T17:18:31.737736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518439613907206:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:31.795549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518439166240796:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21495286ns) {Configure:104(21490105ns) {ProcessConfigured:135(64668ns) {StartInitialHandshake:157(62219ns) {DropHandshakes:469(3867ns) {DropIncomingHandshake:441(829ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(655ns) PrepareNewSessionHandshake:476(2167ns) {SwitchToState:178(870ns)}}}}}
2025-11-11T17:18:31.765694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518438388784362:2061] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.765802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518438953229541:2066] [node 5] ICP01 ready to work
2025-11-11T17:18:31.765672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518440546695412:2065] [node 5] ICP01 ready to work
2025-11-11T17:18:31.826781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518438388784330:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(137155332ns) {Configure:104(137149988ns) {ProcessConfigured:135(75591ns) {StartInitialHandshake:157(72505ns) {DropHandshakes:469(2932ns) {DropIncomingHandshake:441(710ns) DropOutgoingHandshake:456(605ns)} GenerateSessionVirtualId:988(680ns) PrepareNewSessionHandshake:476(1618ns) {SwitchToState:178(469ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.858139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518439166240837:2061] [node 6] ICP01 ready to work
2025-11-11T17:18:31.785784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518440546695413:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.923877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518438388784329:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(250621387ns) {Configure:104(250616534ns) {ProcessConfigured:135(77366ns) {StartInitialHandshake:157(71898ns) {DropHandshakes:469(3360ns) {DropIncomingHandshake:441(744ns) DropOutgoingHandshake:456(544ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:476(2471ns) {SwitchToState:178(1032ns)}}}}}
2025-11-11T17:18:31.924684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518438388784380:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:31.924736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518438388784381:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:31.924834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518438388784391:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:31.825768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518438953229551:2068] [node 6] ICP01 ready to work
2025-11-11T17:18:31.791285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518439613907209:2061] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.975537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518439166240843:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:31.975610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518439166240844:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:31.975759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518439166240845:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:31.977497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518439166240846:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.813161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518440546695427:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42819
2025-11-11T17:18:31.843697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518438953229552:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38981
2025-11-11T17:18:31.934377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518438424493636:2070] [node 5] ICP01 ready to work
2025-11-11T17:18:31.967312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518438388784396:2084] [node 6] ICP01 ready to work
2025-11-11T17:18:31.813212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518440546695428:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42819
2025-11-11T17:18:31.949745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518438424493639:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.014931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518439166240811:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(252189795ns) {Configure:104(252183892ns) {ProcessConfigured:135(83292ns) {StartInitialHandshake:157(80800ns) {DropHandshakes:469(2944ns) {DropIncomingHandshake:441(854ns) DropOutgoingHandshake:456(540ns)} GenerateSessionVirtualId:988(1011ns) PrepareNewSessionHandshake:476(2830ns) {SwitchToState:178(1050ns)}}}}}
2025-11-11T17:18:31.843760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518438953229564:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38981
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.978593Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518438388784391:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.015136Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518439166240844:2087] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:31.858011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518440546695439:2071] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.949924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518438424493640:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:31.950552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518438953229579:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38981
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.102331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518443908874538:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47950
2025-11-11T17:18:31.980078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518438388784329:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.097714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518440546695428:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.016773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518439166240796:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.021637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518438953229566:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.969658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518438424493639:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:31.980164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518438388784399:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:32.047275Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:32.048221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518438388784329:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518438388784391:2105] held: no
2025-11-11T17:18:32.088201Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518438388784380:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.145766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518438388784379:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.145802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518438388784379:2094] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.162906Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518438388784399:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.163151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518442683751706:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:32.178207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518442683751709:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:32.069648Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518439166240845:2088] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.186344Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518438388784391:2105] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:32.165716Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518443908874538:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.098405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518440546695380:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(440212838ns) {Configure:104(440208808ns) {ProcessConfigured:135(60519ns) {StartInitialHandshake:157(58902ns) {DropHandshakes:469(3600ns) {DropIncomingHandshake:441(1031ns) DropOutgoingHandshake:456(464ns)} GenerateSessionVirtualId:988(1099ns) PrepareNewSessionHandshake:476(7301ns) {SwitchToState:178(561ns)}}}}}
2025-11-11T17:18:32.021678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518438953229566:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.086837Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518438424493640:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.171577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518443908874539:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47950
2025-11-11T17:18:32.201805Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518438388784381:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.210358Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518438388784379:2094] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:32.214988Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518442683751709:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.215054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518438388784329:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.215102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:32.215145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518438388784329:2048] [node 1] ICP22 created new session: [4:7571518442683751711:2048]
2025-11-11T17:18:32.215188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518442683751711:2048] [node 1] ICS09 handshake done sender: [4:7571518438388784391:2105] self: [4:7571518438388784402:0] peer: [1:7571518438953229565:0] socket: 131 qp: -1
2025-11-11T17:18:32.215221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518442683751711:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:32.215260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:32.215851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518438388784330:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.218461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518438388784362:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.218501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:32.089795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518439166240811:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.102007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518443461208174:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:32.102718Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518439166240844:2087] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:32.102896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518439166240796:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.102941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:32.102980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518439166240796:2048] [node 1] ICP22 created new session: [5:7571518443461208175:2048]
2025-11-11T17:18:32.103022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518443461208175:2048] [node 1] ICS09 handshake done sender: [5:7571518439166240844:2087] self: [5:7571518443461208157:0] peer: [1:7571518438953229567:0] socket: 141 qp: -1
2025-11-11T17:18:32.103055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518443461208175:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:32.103093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:32.110307Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518439166240839:2083] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.110348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518439166240839:2083] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.132184Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518439166240839:2083] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:32.177660Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518439166240846:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.185655Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518439166240843:2086] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.186704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518439166240798:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(464479299ns) {Configure:104(464471726ns) {ProcessConfigured:135(4760109ns) {StartInitialHandshake:157(4757512ns) {DropHandshakes:469(3410ns) {DropIncomingHandshake:441(1121ns) DropOutgoingHandshake:456(609ns)} GenerateSessionVirtualId:988(1155ns) PrepareNewSessionHandshake:476(2141ns) {SwitchToState:178(994ns)}}}}}
2025-11-11T17:18:32.187005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518439166240837:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.187072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:32.187114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518439166240837:2061] [node 6] ICP22 created new session: [5:7571518443461208181:2061]
2025-11-11T17:18:32.187159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518443461208181:2061] [node 6] ICS09 handshake done sender: [5:7571518439166240839:2083] self: [5:7571518439166240838:0] peer: [6:7571518442719460942:0] socket: 130 qp: -1
2025-11-11T17:18:32.187190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518443461208181:2061] [node 6] ICS10 traffic start
2025-11-11T17:18:32.187222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:32.213393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518439166240797:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.217012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518443461208185:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:32.039030Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518438953229566:2109] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:32.234223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518438388784362:2061] [node 5] ICP22 created new session: [4:7571518442683751715:2061]
2025-11-11T17:18:32.100315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518444841662751:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42819
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.086932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460941:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.183062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518439613907168:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.234278Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518439166240845:2088] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:32.039290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518438953229539:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.039334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:32.039380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518438953229539:2065] [node 4] ICP22 created new session: [1:7571518443248196878:2065]
2025-11-11T17:18:32.039424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518443248196878:2065] [node 4] ICS09 handshake done sender: [1:7571518438953229566:2109] self: [1:7571518438953229565:0] peer: [4:7571518438388784402:0] socket: 127 qp: -1
2025-11-11T17:18:32.039455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518443248196878:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:32.039492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:32.046578Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518438953229579:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.046703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518438953229539:2065] [node 4] ICP18 handshake (actor [1:7571518438953229579:2118]) is too old
2025-11-11T17:18:32.063624Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518438953229552:2096] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.069635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518438953229568:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.077656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518438953229568:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.086167Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518438953229564:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.111128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518438953229568:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:32.141962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518438953229500:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(499877996ns) {Configure:104(499871616ns) {ProcessConfigured:135(19859045ns) {StartInitialHandshake:157(19855510ns) {DropHandshakes:469(3538ns) {DropIncomingHandshake:441(831ns) DropOutgoingHandshake:456(488ns)} GenerateSessionVirtualId:988(1385ns) PrepareNewSessionHandshake:476(3793ns) {SwitchToState:178(1183ns)}}}}}
2025-11-11T17:18:32.163519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518438953229541:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.170180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518438953229501:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(515205470ns) {Configure:104(515200078ns) {ProcessConfigured:135(19530820ns) {StartInitialHandshake:157(19527688ns) {DropHandshakes:469(3203ns) {DropIncomingHandshake:441(862ns) DropOutgoingHandshake:456(633ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:476(3214ns) {SwitchToState:178(930ns)}}}}}
2025-11-11T17:18:32.173217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:32.196084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518443248196891:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38981
2025-11-11T17:18:32.196554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518438953229541:2066] [node 5] ICP22 created new session: [1:7571518443248196890:2066]
2025-11-11T17:18:32.201647Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518438953229570:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.201685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518438953229570:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.209767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518443248196891:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.226263Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518438953229570:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:32.233924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518443248196890:2066] [node 5] ICS09 handshake done sender: [1:7571518438953229568:2110] self: [1:7571518438953229567:0] peer: [5:7571518443461208157:0] socket: 128 qp: -1
2025-11-11T17:18:32.233969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518443248196890:2066] [node 5] ICS10 traffic start
2025-11-11T17:18:32.249687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:32.273846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518442683751715:2061] [node 5] ICS09 handshake done sender: [4:7571518438388784379:2094] self: [4:7571518438388784378:0] peer: [5:7571518443461208172:0] socket: 132 qp: -1
2025-11-11T17:18:32.254215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518443908874542:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47950
2025-11-11T17:18:32.254377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518443908874541:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47950
2025-11-11T17:18:32.254873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518443908874538:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:32.255058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518439613907168:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.255144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:32.255190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518439613907168:2049] [node 2] ICP22 created new session: [3:7571518443908874544:2049]
2025-11-11T17:18:32.255231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518443908874544:2049] [node 2] ICS09 handshake done sender: [3:7571518443908874538:2113] self: [3:7571518443908874540:0] peer: [2:7571518444841662749:0] socket: 151 qp: -1
2025-11-11T17:18:32.255268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518443908874544:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:32.255305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:32.269908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518439613907206:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(483913306ns) {Configure:104(483906497ns) {ProcessConfigured:135(19665964ns) {StartInitialHandshake:157(19663063ns) {DropHandshakes:469(873ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(346ns) PrepareNewSessionHandshake:476(2476ns) {SwitchToState:178(1360ns)}}}}}
2025-11-11T17:18:32.274939Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518443908874542:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.278070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518442683751715:2061] [node 5] ICS10 traffic start
2025-11-11T17:18:32.278139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:32.130094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518440546695427:2093] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.087052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518438424493636:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.278142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518438953229551:2068] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.249891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518439166240837:2061] [node 6] ICP18 handshake (actor [5:7571518439166240846:2089]) is too old
2025-11-11T17:18:32.146164Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518444841662751:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.302007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518438388784401:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.289950Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518443908874539:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.270535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518439166240811:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.126423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518438424493573:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(468436915ns) {Configure:104(463671823ns) {ProcessConfigured:135(76426ns) {StartInitialHandshake:157(73709ns) {DropHandshakes:469(5249ns) {DropIncomingHandshake:441(652ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(1063ns) PrepareNewSessionHandshake:476(2108ns) {SwitchToState:178(661ns)}}}}}
2025-11-11T17:18:32.302055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518438388784401:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.305818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518439613907167:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.146503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518444841662754:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42819
2025-11-11T17:18:32.298363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:32.146674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518444841662756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42819
2025-11-11T17:18:32.161911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518440546695379:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(539910148ns) {Configure:104(539901552ns) {ProcessConfigured:135(18888773ns) {StartInitialHandshake:157(18884280ns) {DropHandshakes:469(3278ns) {DropIncomingHandshake:441(766ns) DropOutgoingHandshake:456(591ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:476(4469ns) {SwitchToState:178(1969ns)}}}}}
2025-11-11T17:18:32.162106Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518444841662754:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.191753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518444841662750:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.191796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518444841662750:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.212613Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518444841662750:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:32.265929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518440546695439:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(344006846ns) {Configure:104(343998436ns) {ProcessConfigured:135(15924178ns) {StartInitialHandshake:157(15921183ns) {DropHandshakes:469(3426ns) {DropIncomingHandshake:441(776ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(586ns) PrepareNewSessionHandshake:476(3856ns) {SwitchToState:178(1158ns)}}}}}
2025-11-11T17:18:32.309679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518444841662756:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.309904Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518440546695432:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.309929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518440546695432:2096] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.310463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518440546695379:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.329742Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518440546695430:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.329784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518440546695430:2095] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.329991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518440546695380:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.330058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:32.341889Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518440546695432:2096] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:32.343822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518440546695413:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.343910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:32.343959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518440546695413:2066] [node 4] ICP22 created new session: [2:7571518444841662769:2066]
2025-11-11T17:18:32.344007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518444841662769:2066] [node 4] ICS09 handshake done sender: [2:7571518440546695432:2096] self: [2:7571518440546695431:0] peer: [4:7571518442683751713:0] socket: 124 qp: -1
2025-11-11T17:18:32.344042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518444841662769:2066] [node 4] ICS10 traffic start
2025-11-11T17:18:32.344105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:32.350510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518444841662767:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42819
2025-11-11T17:18:32.353693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518440546695380:2049] [node 3] ICP22 created new session: [2:7571518444841662766:2049]
2025-11-11T17:18:32.356775Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518440546695430:2095] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:32.359957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518440546695412:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.360013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:32.360055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518440546695412:2065] [node 5] ICP22 created new session: [2:7571518444841662773:2065]
2025-11-11T17:18:32.360111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518444841662773:2065] [node 5] ICS09 handshake done sender: [2:7571518440546695430:2095] self: [2:7571518440546695429:0] peer: [5:7571518443461208184:0] socket: 123 qp: -1
2025-11-11T17:18:32.360142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518444841662773:2065] [node 5] ICS10 traffic start
2025-11-11T17:18:32.360172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:32.373801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518444841662766:2049] [node 3] ICS09 handshake done sender: [2:7571518444841662750:2114] self: [2:7571518444841662749:0] peer: [3:7571518443908874540:0] socket: 148 qp: -1
2025-11-11T17:18:32.386735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518444841662766:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:32.386784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.270587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:32.129079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460946:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.309819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518438953229551:2068] [node 6] ICP22 created new session: [1:7571518443248196895:2068]
2025-11-11T17:18:32.312337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518442683751720:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:32.330658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518443248196887:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.330676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518443248196887:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.331166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518443248196884:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.331184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518443248196884:2120] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.334008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518443248196895:2068] [node 6] ICS09 handshake done sender: [1:7571518438953229570:2111] self: [1:7571518438953229569:0] peer: [6:7571518442719460951:0] socket: 129 qp: -1
2025-11-11T17:18:32.351302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518443248196895:2068] [node 6] ICS10 traffic start
2025-11-11T17:18:32.351353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:32.351653Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518443248196884:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:32.357054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518443248196887:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:32.359067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518438953229541:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(195327993ns) {HandleHandshakeStatus:329(195324942ns) {DropIncomingHandshake:441(19507ns) DropOutgoingHandshake:456(393ns) SwitchToState:178(962ns) ProcessPendingSessionEvents:495(32531537ns) {ForwardSessionEventToSession:612(32475870ns) ForwardSessionEventToSession:612(4730ns) ForwardSessionEventToSession:612(5957ns) ForwardSessionEventToSession:612(2609ns) ForwardSessionEventToSession:612(2795ns) ForwardSessionEventToSession:612(13154ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(7159ns) ForwardSessionEventToSession:612(2132ns) ForwardSessionEventToSession:612(2724ns)}}}
2025-11-11T17:18:32.359667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518443248196899:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38981
2025-11-11T17:18:32.391464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518438953229500:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.391516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:32.398162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518438953229501:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.398273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:32.398318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518438953229501:2049] [node 3] ICP22 created new session: [1:7571518443248196901:2049]
2025-11-11T17:18:32.398357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518443248196901:2049] [node 3] ICS09 handshake done sender: [1:7571518443248196887:2121] self: [1:7571518443248196886:0] peer: [3:7571518443908874550:0] socket: 159 qp: -1
2025-11-11T17:18:32.398392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518443248196901:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:32.398423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:32.398965Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518443248196899:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.399763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518438953229500:2048] [node 2] ICP22 created new session: [1:7571518443248196900:2048]
2025-11-11T17:18:32.306427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518443908874551:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47950
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.391260Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518444841662756:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:32.158427Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518438424493640:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:32.312818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518439613907209:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(458153616ns) {Configure:104(458145173ns) {ProcessConfigured:135(10752042ns) {StartInitialHandshake:157(10749163ns) {DropHandshakes:469(3129ns) {DropIncomingHandshake:441(702ns) DropOutgoingHandshake:456(719ns)} GenerateSessionVirtualId:988(1153ns) PrepareNewSessionHandshake:476(3054ns) {SwitchToState:178(1281ns)}}}}}
2025-11-11T17:18:32.270645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518439166240811:2060] [node 4] ICP22 created new session: [5:7571518443461208186:2060]
2025-11-11T17:18:32.325980Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518438388784401:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:32.417531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518443248196900:2048] [node 2] ICS09 handshake done sender: [1:7571518443248196884:2120] self: [1:7571518443248196883:0] peer: [2:7571518444841662764:0] socket: 156 qp: -1
2025-11-11T17:18:32.321935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518443908874553:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47950
2025-11-11T17:18:32.330242Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518443908874551:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.333456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518443908874555:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47950
2025-11-11T17:18:32.337762Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518443908874553:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.374294Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518443908874546:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.374319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518443908874546:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.374990Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518443908874539:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:32.375382Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518443908874546:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:32.375577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518439613907206:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.375635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:32.375678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518439613907206:2062] [node 5] ICP22 created new session: [3:7571518443908874558:2062]
2025-11-11T17:18:32.375720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518443908874558:2062] [node 5] ICS09 handshake done sender: [3:7571518443908874546:2117] self: [3:7571518443908874543:0] peer: [5:7571518443461208189:0] socket: 114 qp: -1
2025-11-11T17:18:32.375749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518443908874558:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:32.375780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:32.385941Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518439613907195:2071] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.385965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518439613907195:2071] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.391511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518439613907167:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.417756Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518439613907195:2071] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:32.426374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:32.443102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518439613907192:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.458069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:32.394905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518440546695379:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.270688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518443461208186:2060] [node 4] ICS09 handshake done sender: [5:7571518439166240845:2088] self: [5:7571518443461208172:0] peer: [4:7571518438388784378:0] socket: 142 qp: -1
2025-11-11T17:18:32.469853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518439613907167:2048] [node 1] ICP22 created new session: [3:7571518443908874562:2048]
2025-11-11T17:18:32.422169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518438953229551:2068] [node 6] ICP35 event processing took too much time PendingConnection:278(143753899ns) {HandleHandshakeStatus:329(143750921ns) {DropIncomingHandshake:441(13475ns) DropOutgoingHandshake:456(541ns) SwitchToState:178(751ns) ProcessPendingSessionEvents:495(34423517ns) {ForwardSessionEventToSession:612(22468633ns) ForwardSessionEventToSession:612(11905071ns) ForwardSessionEventToSession:612(3336ns) ForwardSessionEventToSession:612(5106ns) ForwardSessionEventToSession:612(2414ns) ForwardSessionEventToSession:612(2971ns) ForwardSessionEventToSession:612(10360ns) ForwardSessionEventToSession:612(2363ns) ForwardSessionEventToSession:612(2357ns) ForwardSessionEventToSession:612(7328ns)}}}
2025-11-11T17:18:32.442688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518443248196900:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:32.442753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:32.478651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518438953229500:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(91542575ns) {HandleHandshakeStatus:329(91540315ns) {DropIncomingHandshake:441(1652ns) DropOutgoingHandshake:456(367ns) SwitchToState:178(749ns) ProcessPendingSessionEvents:495(8697080ns) {ForwardSessionEventToSession:612(8639182ns) ForwardSessionEventToSession:612(8568ns) ForwardSessionEventToSession:612(1969ns) ForwardSessionEventToSession:612(2037ns) ForwardSessionEventToSession:612(5154ns) ForwardSessionEventToSession:612(3258ns) ForwardSessionEventToSession:612(5353ns) ForwardSessionEventToSession:612(2394ns) ForwardSessionEventToSession:612(7773ns) ForwardSessionEventToSession:612(4526ns)}}}
2025-11-11T17:18:32.478737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518438953229500:2048] [node 2] ICP18 handshake (actor [1:7571518443248196899:2123]) is too old
2025-11-11T17:18:32.473891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518439613907192:2067] [node 4] ICP22 created new session: [3:7571518443908874563:2067]
2025-11-11T17:18:32.158618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518438424493636:2070] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.332546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518438388784362:2061] [node 5] ICP35 event processing took too much time PendingConnection:278(113880189ns) {HandleHandshakeStatus:329(113878695ns) {DropIncomingHandshake:441(10069ns) DropOutgoingHandshake:456(684ns) SwitchToState:178(152ns) ProcessPendingSessionEvents:495(18267565ns) {ForwardSessionEventToSession:612(18215367ns) ForwardSessionEventToSession:612(10137ns) ForwardSessionEventToSession:612(4972ns) ForwardSessionEventToSession:612(4690ns) ForwardSessionEventToSession:612(2106ns) ForwardSessionEventToSession:612(3110ns) ForwardSessionEventToSession:612(5843ns) ForwardSessionEventToSession:612(2492ns) ForwardSessionEventToSession:612(3102ns) ForwardSessionEventToSession:612(5881ns)}}}
2025-11-11T17:18:32.357695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518438388784331:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(632380683ns) {Configure:104(632376526ns) {ProcessConfigured:135(16169006ns) {StartInitialHandshake:157(16165875ns) {DropHandshakes:469(3131ns) {DropIncomingHandshake:441(590ns) DropOutgoingHandshake:456(589ns)} GenerateSessionVirtualId:988(884ns) PrepareNewSessionHandshake:476(4108ns) {SwitchToState:178(2025ns)}}}}}
2025-11-11T17:18:32.361765Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:32.382941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518438388784396:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.270720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518443461208186:2060] [node 4] ICS10 traffic start
2025-11-11T17:18:32.394971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:32.383010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:32.383055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518438388784396:2084] [node 6] ICP22 created new session: [4:7571518442683751722:2084]
2025-11-11T17:18:32.383095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518442683751722:2084] [node 6] ICS09 handshake done sender: [4:7571518438388784401:2111] self: [4:7571518438388784400:0] peer: [6:7571518442719460956:0] socket: 144 qp: -1
2025-11-11T17:18:32.383124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518442683751722:2084] [node 6] ICS10 traffic start
2025-11-11T17:18:32.383158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:32.383603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518438388784331:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.387331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518438388784330:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518438388784381:2096] held: no
2025-11-11T17:18:32.417613Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518438388784381:2096] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:32.470053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518438388784330:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.470121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:32.470168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518438388784330:2049] [node 2] ICP22 created new session: [4:7571518442683751726:2049]
2025-11-11T17:18:32.470206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518442683751726:2049] [node 2] ICS09 handshake done sender: [4:7571518438388784381:2096] self: [4:7571518442683751713:0] peer: [2:7571518440546695431:0] socket: 126 qp: -1
2025-11-11T17:18:32.470246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518442683751726:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:32.470288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:32.470468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518438388784330:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20254821ns) {HandleHandshakeStatus:329(20251214ns) {DropIncomingHandshake:441(2039ns) DropOutgoingHandshake:456(541ns) SwitchToState:178(568ns) ProcessPendingSessionEvents:495(269ns)}}
2025-11-11T17:18:32.481816Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:32.482612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518438388784331:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518438388784380:2095] held: no
2025-11-11T17:18:32.481151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518443908874562:2048] [node 1] ICS09 handshake done sender: [3:7571518443908874539:2114] self: [3:7571518443908874550:0] peer: [1:7571518443248196886:0] socket: 161 qp: -1
2025-11-11T17:18:32.158666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:32.395030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518440546695379:2048] [node 1] ICP22 created new session: [2:7571518444841662776:2048]
2025-11-11T17:18:32.395088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518444841662776:2048] [node 1] ICS09 handshake done sender: [2:7571518444841662756:2117] self: [2:7571518444841662764:0] peer: [1:7571518443248196883:0] socket: 158 qp: -1
2025-11-11T17:18:32.395129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518444841662776:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:32.395161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:32.479740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518440546695380:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(165369356ns) {HandleHandshakeStatus:329(149541956ns) {DropIncomingHandshake:441(14715ns) DropOutgoingHandshake:456(470ns) SwitchToState:178(975ns) ProcessPendingSessionEvents:495(33805431ns) {ForwardSessionEventToSession:612(17209088ns) ForwardSessionEventToSession:612(16552511ns) ForwardSessionEventToSession:612(2060ns) ForwardSessionEventToSession:612(5711ns) ForwardSessionEventToSession:612(2127ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(10454ns) ForwardSessionEventToSession:612(2732ns) ForwardSessionEventToSession:612(2312ns) ForwardSessionEventToSession:612(3951ns)}}}
2025-11-11T17:18:32.502032Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518444841662762:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.502076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518444841662762:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.512459Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518444841662762:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:32.482938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518443908874563:2067] [node 4] ICS09 handshake done sender: [3:7571518439613907195:2071] self: [3:7571518439613907194:0] peer: [4:7571518442683751724:0] socket: 115 qp: -1
2025-11-11T17:18:32.270753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:32.158714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518438424493636:2070] [node 5] ICP22 created new session: [6:7571518442719460948:2070]
2025-11-11T17:18:32.482973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518443908874563:2067] [node 4] ICS10 traffic start
2025-11-11T17:18:32.489643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518443908874562:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:32.490707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518443908874552:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.490732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518443908874552:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:32.491973Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518443908874552:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:32.492228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518439613907209:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.492299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:32.492370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518439613907209:2061] [node 6] ICP22 created new session: [3:7571518443908874565:2061]
2025-11-11T17:18:32.492421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518443908874565:2061] [node 6] ICS09 handshake done sender: [3:7571518443908874552:2119] self: [3:7571518443908874549:0] peer: [6:7571518442719460964:0] socket: 181 qp: -1
2025-11-11T17:18:32.492476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518443908874565:2061] [node 6] ICS10 traffic start
2025-11-11T17:18:32.492514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:32.505539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:32.513846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:32.529937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518440546695439:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:32.271578Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64334) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:32.278001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518439166240811:2060] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:32.297952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518443461208188:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:32.353742Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518443461208188:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.358656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518443461208190:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:32.362655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518439166240798:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.369801Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518439166240843:2086] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:32.382220Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:32.384306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518439166240798:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518443461208188:2121] held: no
2025-11-11T17:18:32.387370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518439166240797:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.387422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:32.397709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518443461208192:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:18:32.402024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518439166240797:2049] [node 2] ICP22 created new session: [5:7571518443461208191:2049]
2025-11-11T17:18:32.426272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518443461208191:2049] [node 2] ICS09 handshake done sender: [5:7571518439166240843:2086] self: [5:7571518443461208184:0] peer: [2:7571518440546695429:0] socket: 140 qp: -1
2025-11-11T17:18:32.426314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518443461208191:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:32.429695Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518443461208188:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:32.429976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518439166240798:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.437612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:32.442844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:32.461665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518439166240798:2050] [node 3] ICP22 created new session: [5:7571518443461208193:2050]
2025-11-11T17:18:32.477680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518443461208193:2050] [node 3] ICS09 handshake done sender: [5:7571518443461208188:2121] self: [5:7571518443461208189:0] peer: [3:7571518443908874543:0] socket: 157 qp: -1
2025-11-11T17:18:32.477737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518443461208193:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:32.477797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:32.478039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518439166240798:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(47960656ns) {HandleHandshakeStatus:329(47957572ns) {DropIncomingHandshake:441(1442ns) DropOutgoingHandshake:456(502ns) SwitchToState:178(642ns) ProcessPendingSessionEvents:495(1414ns)}}
2025-11-11T17:18:32.550089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518439613907167:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(171483906ns) {HandleHandshakeStatus:329(171480834ns) {DropIncomingHandshake:441(2358ns) DropOutgoingHandshake:456(12596ns) SwitchToState:178(433ns) ProcessPendingSessionEvents:495(8054755ns) {ForwardSessionEventToSession:612(7981989ns) ForwardSessionEventToSession:612(12271ns) ForwardSessionEventToSession:612(2606ns) ForwardSessionEventToSession:612(2751ns) ForwardSessionEventToSession:612(4719ns) ForwardSessionEventToSession:612(2059ns) ForwardSessionEventToSession:612(1943ns) ForwardSessionEventToSession:612(4260ns) ForwardSessionEventToSession:612(22238ns) ForwardSessionEventToSession:612(8635ns)}}}
2025-11-11T17:18:32.158754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518442719460948:2070] [node 5] ICS09 handshake done sender: [6:7571518438424493640:2112] self: [6:7571518442719460942:0] peer: [5:7571518439166240838:0] socket: 136 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:32.555064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:32.158793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518442719460948:2070] [node 5] ICS10 traffic start
2025-11-11T17:18:32.158829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:32.170792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518438424493572:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(491353116ns) {Configure:104(491349328ns) {ProcessConfigured:135(16992584ns) {StartInitialHandshake:157(16989843ns) {DropHandshakes:469(3176ns) {DropIncomingHandshake:441(626ns) DropOutgoingHandshake:456(567ns)} GenerateSessionVirtualId:988(755ns) PrepareNewSessionHandshake:476(1976ns) {SwitchToState:178(815ns)}}}}}
2025-11-11T17:18:32.170954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518438424493572:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.182566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518438424493574:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(244916954ns) {Configure:104(244913344ns) {ProcessConfigured:135(9322256ns) {StartInitialHandshake:157(9319740ns) {DropHandshakes:469(5644ns) {DropIncomingHandshake:441(360ns) DropOutgoingHandshake:456(2180ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:476(3089ns) {SwitchToState:178(1164ns)}}}}}
2025-11-11T17:18:32.216298Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518442719460941:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.246426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518438424493594:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.250335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460955:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.285987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518438424493639:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:32.286805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518438424493572:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.286858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:32.286904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518438424493572:2048] [node 1] ICP22 created new session: [6:7571518442719460957:2048]
2025-11-11T17:18:32.286997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518442719460957:2048] [node 1] ICS09 handshake done sender: [6:7571518438424493639:2111] self: [6:7571518442719460951:0] peer: [1:7571518438953229569:0] socket: 135 qp: -1
2025-11-11T17:18:32.287039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518442719460957:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:32.287075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:32.356307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460960:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.390768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518442719460941:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:32.393619Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64334) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:32.401727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460961:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.410902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518438424493594:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64334) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518442719460941:2114] held: no
2025-11-11T17:18:32.410989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518438424493594:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.425915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460962:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.446050Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518442719460962:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.446333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518438424493574:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.449859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:32.457908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518438424493594:2067] [node 4] ICP22 created new session: [6:7571518442719460963:2067]
2025-11-11T17:18:32.469945Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518442719460961:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:32.474062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518438424493573:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:32.477739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518442719460963:2067] [node 4] ICS09 handshake done sender: [6:7571518442719460941:2114] self: [6:7571518442719460956:0] peer: [4:7571518438388784400:0] socket: 145 qp: -1
2025-11-11T17:18:32.477786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518442719460963:2067] [node 4] ICS10 traffic start
2025-11-11T17:18:32.477848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:32.478078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518438424493594:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(66992260ns) {HandleHandshakeStatus:329(66991017ns) {DropIncomingHandshake:441(725ns) DropOutgoingHandshake:456(138ns) SwitchToState:178(585ns) ProcessPendingSessionEvents:495(1191ns)}}
2025-11-11T17:18:32.493575Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:32.510463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460967:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.511931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518442719460968:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-11T17:18:32.512977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518442719460961:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:32.513175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518438424493573:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.513224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:32.513282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518438424493573:2049] [node 2] ICP22 created new session: [6:7571518442719460969:2049]
2025-11-11T17:18:32.513325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518442719460969:2049] [node 2] ICS09 handshake done sender: [6:7571518442719460961:2121] self: [6:7571518442719460965:0] peer: [2:7571518444841662761:0] socket: 179 qp: -1
2025-11-11T17:18:32.513359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518442719460969:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:32.513390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:32.506501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518438424493574:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518442719460962:2122] held: no
2025-11-11T17:18:32.525928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518438424493574:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(31881309ns) {HandleHandshakeStatus:395(20276165ns) {LogHandshakeFail:472(23953ns) {UpdateErrorStateLog:544(6635ns)} DropOutgoingHandshake:456(2131ns) ProcessPendingSessionEvents:495(7858041ns) {DropSessionEvent:512(12879ns) DropSessionEvent:512(3872ns) DropSessionEvent:512(2851ns) DropSessionEvent:512(3469ns) DropSessionEvent:512(3639ns) DropSessionEvent:512(3718ns) DropSessionEvent:512(4105ns) DropSessionEvent:512(7808536ns) DropSessionEvent:512(4795ns) DropSessionEvent:512(3957ns)}}}
2025-11-11T17:18:32.526040Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518442719460962:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:32.530158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518438424493574:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.545884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:32.554988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518438424493574:2050] [node 3] ICP22 created new session: [6:7571518442719460971:2050]
2025-11-11T17:18:32.561728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518442719460971:2050] [node 3] ICS09 handshake done sender: [6:7571518442719460962:2122] self: [6:7571518442719460964:0] peer: [3:7571518443908874549:0] socket: 183 qp: -1
2025-11-11T17:18:32.565828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518442719460971:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:32.573803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:32.580421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518439613907192:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(130623710ns) {HandleHandshakeStatus:329(130619630ns) {DropIncomingHandshake:441(14272ns) DropOutgoingHandshake:456(653ns) SwitchToState:178(851ns) ProcessPendingSessionEvents:495(27979954ns) {ForwardSessionEventToSession:612(7891710ns) ForwardSessionEventToSession:612(18342ns) ForwardSessionEventToSession:612(3765ns) ForwardSessionEventToSession:612(3997034ns) ForwardSessionEventToSession:612(4749ns) ForwardSessionEventToSession:612(2552ns) ForwardSessionEventToSession:612(463962ns) ForwardSessionEventToSession:612(3614ns) ForwardSessionEventToSession:612(15542196ns) ForwardSessionEventToSession:612(23390ns)}}}
2025-11-11T17:18:32.570127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518440546695439:2071] [node 6] ICP22 created new session: [2:7571518444841662779:2071]
2025-11-11T17:18:32.574067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518444841662779:2071] [node 6] ICS09 handshake done sender: [2:7571518444841662762:2119] self: [2:7571518444841662761:0] peer: [6:7571518442719460965:0] socket: 177 qp: -1
2025-11-11T17:18:32.574116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518444841662779:2071] [node 6] ICS10 traffic start
2025-11-11T17:18:32.574172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:32.579468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518440546695439:2071] [node 6] ICP35 event processing took too much time PendingConnection:278(56756873ns) {HandleHandshakeStatus:329(56752833ns) {DropIncomingHandshake:441(24429ns) DropOutgoingHandshake:456(528ns) SwitchToState:178(883ns) ProcessPendingSessionEvents:495(4916889ns) {ForwardSessionEventToSession:612(4871424ns) ForwardSessionEventToSession:612(4083ns) ForwardSessionEventToSession:612(5464ns) ForwardSessionEventToSession:612(2920ns) ForwardSessionEventToSession:612(7885ns) ForwardSessionEventToSession:612(4256ns) ForwardSessionEventToSession:612(1842ns) ForwardSessionEventToSession:612(2068ns) ForwardSessionEventToSession:612(5153ns) ForwardSessionEventToSession:612(2544ns)}}}
2025-11-11T17:18:32.581099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518439166240797:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(156664456ns) {HandleHandshakeStatus:329(156661425ns) {DropIncomingHandshake:441(756ns) DropOutgoingHandshake:456(10272ns) SwitchToState:178(332ns) ProcessPendingSessionEvents:495(48533819ns) {ForwardSessionEventToSession:612(19949973ns) ForwardSessionEventToSession:612(6167ns) ForwardSessionEventToSession:612(4140694ns) ForwardSessionEventToSession:612(7875333ns) ForwardSessionEventToSession:612(8348378ns) ForwardSessionEventToSession:612(6269ns) ForwardSessionEventToSession:612(3712ns) ForwardSessionEventToSession:612(8169044ns) ForwardSessionEventToSession:612(5740ns) ForwardSessionEventToSession:612(2723ns)}}}
2025-11-11T17:18:32.574421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518442683751725:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20219
2025-11-11T17:18:32.583060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518438388784380:2095] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:32.583358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518438388784331:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:32.583406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:32.583444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518438388784331:2050] [node 3] ICP22 created new session: [4:7571518442683751728:2050]
2025-11-11T17:18:32.583487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518442683751728:2050] [node 3] ICS09 handshake done sender: [4:7571518438388784380:2095] self: [4:7571518442683751724:0] peer: [3:7571518439613907194:0] socket: 125 qp: -1
2025-11-11T17:18:32.583515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518442683751728:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:32.583550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:32.582261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518438424493574:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(52032719ns) {HandleHandshakeStatus:329(52029959ns) {DropIncomingHandshake:441(315ns) DropOutgoingHandshake:456(203ns) SwitchToState:178(346ns) ProcessPendingSessionEvents:495(1373ns)}}
2025-11-11T17:18:32.637059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:32.637293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518444841662776:2048] [node 1] ICS01 socket: 158 reason# EndOfStream
2025-11-11T17:18:32.637325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518440546695379:2048] [node 1] ICP30 unregister session Session# [2:7571518444841662776:2048] VirtualId# [2:7571518444841662764:0]
2025-11-11T17:18:32.637353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518444841662776:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.637415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:32.637456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:32.639334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:32.639557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518443908874562:2048] [node 1] ICS07 socket disconnect 161 reason# EndOfStream
2025-11-11T17:18:32.639579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518443908874562:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.639637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:32.639667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518443908874562:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:32.646341Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:18:32.647296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518439613907167:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:18:32.647329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518443908874562:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:32.647978Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Couldn't connect to any resolved address
2025-11-11T17:18:32.648201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518439613907167:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:32.648251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518443908874562:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:32.648280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518439613907167:2048] [node 1] ICP30 unregister session Session# [3:7571518443908874562:2048] VirtualId# [3:7571518443908874550:0]
2025-11-11T17:18:32.648332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:32.648362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518439613907167:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:32.648376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17816) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:32.657864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518439613907167:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:32.661772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:32.662000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518442719460957:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-11T17:18:32.662036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518438424493572:2048] [node 1] ICP30 unregister session Session# [6:7571518442719460957:2048] VirtualId# [6:7571518442719460951:0]
2025-11-11T17:18:32.661796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:32.662068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518442719460957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.662140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:32.662185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:32.662004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518443461208175:2048] [node 1] ICS01 socket: 141 reason# EndOfStream
2025-11-11T17:18:32.689813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:32.694058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518442683751711:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:18:32.694114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518438388784329:2048] [node 1] ICP30 unregister session Session# [4:7571518442683751711:2048] VirtualId# [4:7571518438388784402:0]
2025-11-11T17:18:32.694145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518442683751711:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.694253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:32.694296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:32.662058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518439166240796:2048] [node 1] ICP30 unregister session Session# [5:7571518443461208175:2048] VirtualId# [5:7571518443461208157:0]
2025-11-11T17:18:32.662084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518443461208175:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.662176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:32.662216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:32.727217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:32.727444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518443908874544:2049] [node 2] ICS01 socket: 151 reason# EndOfStream
2025-11-11T17:18:32.727493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518439613907168:2049] [node 2] ICP30 unregister session Session# [3:7571518443908874544:2049] VirtualId# [3:7571518443908874540:0]
2025-11-11T17:18:32.727531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518443908874544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.727595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:32.727646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:32.737744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:32.737743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:32.737981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518443461208191:2049] [node 2] ICS07 socket disconnect 140 reason# EndOfStream
2025-11-11T17:18:32.738008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518443461208191:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.738087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:32.738130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518443461208191:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:32.739557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518442683751726:2049] [node 2] ICS01 socket: 126 reason# EndOfStream
2025-11-11T17:18:32.739594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518438388784330:2049] [node 2] ICP30 unregister session Session# [4:7571518442683751726:2049] VirtualId# [4:7571518442683751713:0]
2025-11-11T17:18:32.739612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518442683751726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.739655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:32.739686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:32.740096Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17319) Couldn't connect to any resolved address
2025-11-11T17:18:32.740328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518439166240797:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:32.740369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518443461208191:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:32.740400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518439166240797:2049] [node 2] ICP30 unregister session Session# [5:7571518443461208191:2049] VirtualId# [5:7571518443461208184:0]
2025-11-11T17:18:32.740436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:32.740462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518439166240797:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:32.740475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:32.738649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:32.738904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518442719460969:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-11-11T17:18:32.738947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518438424493573:2049] [node 2] ICP30 unregister session Session# [6:7571518442719460969:2049] VirtualId# [6:7571518442719460965:0]
2025-11-11T17:18:32.738983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518442719460969:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.739032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:32.739073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:32.749698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518439166240797:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:32.865738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:32.865984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518443461208193:2050] [node 3] ICS01 socket: 157 reason# EndOfStream
2025-11-11T17:18:32.866017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518439166240798:2050] [node 3] ICP30 unregister session Session# [5:7571518443461208193:2050] VirtualId# [5:7571518443461208189:0]
2025-11-11T17:18:32.866045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518443461208193:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.866117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:32.866156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:32.865738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:32.865984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518442683751728:2050] [node 3] ICS01 socket: 125 reason# EndOfStream
2025-11-11T17:18:32.866026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518438388784331:2050] [node 3] ICP30 unregister session Session# [4:7571518442683751728:2050] VirtualId# [4:7571518442683751724:0]
2025-11-11T17:18:32.866050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518442683751728:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.866124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:32.866157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:32.866231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:32.866404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518442719460971:2050] [node 3] ICS01 socket: 183 reason# EndOfStream
2025-11-11T17:18:32.866433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518438424493574:2050] [node 3] ICP30 unregister session Session# [6:7571518442719460971:2050] VirtualId# [6:7571518442719460964:0]
2025-11-11T17:18:32.866450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518442719460971:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.866487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:32.866513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:32.949766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:32.950012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518443461208186:2060] [node 4] ICS01 socket: 142 reason# EndOfStream
2025-11-11T17:18:32.950051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518439166240811:2060] [node 4] ICP30 unregister session Session# [5:7571518443461208186:2060] VirtualId# [5:7571518443461208172:0]
2025-11-11T17:18:32.950082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518443461208186:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.950158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:32.950197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:32.953735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:32.953962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518442719460963:2067] [node 4] ICS01 socket: 145 reason# EndOfStream
2025-11-11T17:18:32.954003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518438424493594:2067] [node 4] ICP30 unregister session Session# [6:7571518442719460963:2067] VirtualId# [6:7571518442719460956:0]
2025-11-11T17:18:32.954040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518442719460963:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:32.954143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:32.954189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:33.077744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:33.078004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518442719460948:2070] [node 5] ICS01 socket: 136 reason# EndOfStream
2025-11-11T17:18:33.078045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518438424493636:2070] [node 5] ICP30 unregister session Session# [6:7571518442719460948:2070] VirtualId# [6:7571518442719460942:0]
2025-11-11T17:18:33.078080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518442719460948:2070] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:33.078213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:33.078269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:33.297940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518445947693609:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:33.297988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518445947693610:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:33.341929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518446907538046:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:33.341974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518446907538047:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:33.395328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518447500640518:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:33.395371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518447500640519:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:33.474128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518447380118166:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:33.474169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518447380118167:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:33.474193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518447380118168:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:33.596116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518446231343505:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:33.596159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518446231343506:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:33.596181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518446231343507:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:33.831351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518446231343518:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:33.847243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518447380118179:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:33.850988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518447500640543:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:33.851590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518445947693638:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35702
2025-11-11T17:18:33.897739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518446907538066:2061] [node 4] ICP01 ready to work
2025-11-11T17:18:33.867857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518445947693637:2065] [node 5] ICP01 ready to work
2025-11-11T17:18:33.869631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518447500640540:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:33.941388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518445947693655:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35702
2025-11-11T17:18:33.966218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518446907538079:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:33.929299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518445947693646:2062] [node 4] ICP01 ready to work
2025-11-11T17:18:33.914441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518448289889910:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:33.914492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518448289889911:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:33.914556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518448289889912:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:33.915215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518448289889928:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:33.915458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518448289889929:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:33.946445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518447500640545:2064] [node 5] ICP01 ready to work
2025-11-11T17:18:34.014398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518446231343518:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(182723406ns) {Configure:104(182717261ns) {ProcessConfigured:135(52426677ns) {StartInitialHandshake:157(52423161ns) {DropHandshakes:469(3828ns) {DropIncomingHandshake:441(756ns) DropOutgoingHandshake:456(1196ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:476(5190ns) {SwitchToState:178(2098ns)}}}}}
2025-11-11T17:18:33.943066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518445947693651:2063] [node 6] ICP01 ready to work
2025-11-11T17:18:34.017171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518451795607850:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34814
2025-11-11T17:18:34.017332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518451795607851:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34814
2025-11-11T17:18:34.093808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607850:2078] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.166679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518451202505409:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.178024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518448289889912:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(260838645ns) {Configure:104(260831395ns) {ProcessConfigured:135(87391ns) {StartInitialHandshake:157(83474ns) {DropHandshakes:469(3355ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(664ns)} GenerateSessionVirtualId:988(1034ns) PrepareNewSessionHandshake:476(1992ns) {SwitchToState:178(581ns)}}}}}
2025-11-11T17:18:34.197640Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13023) Couldn't connect to any resolved address
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.145133Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518445947693655:2088] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.021694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518446231343531:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:34.257712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518445947693640:2076] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.257768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518445947693640:2076] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.215866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857284:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.276205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505409:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.170170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310849:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.225924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518447500640518:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.238446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518447380118179:2059] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13023) Couldn't connect to any resolved address incoming: [0:0:0] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.282620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518447500640518:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(77432890ns) {IncomingHandshake:261(34186119ns) {DropIncomingHandshake:441(607ns) PrepareNewSessionHandshake:476(2764ns) {SwitchToState:178(1392ns)} GenerateSessionVirtualId:988(721ns) IssueIncomingHandshakeReply:193(34136312ns) {SwitchToState:178(2848ns)}}}
2025-11-11T17:18:34.283886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607851:2079] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.284448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518451795607889:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34814
2025-11-11T17:18:34.284593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518451795607890:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34814
2025-11-11T17:18:34.289956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607889:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.244523Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518452584857284:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.170224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310850:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.294041Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518445947693640:2076] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.269259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085535:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
2025-11-11T17:18:34.244674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518448289889928:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.170263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310851:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.298671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518446907538047:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(364659038ns) {Configure:104(364655377ns) {ProcessConfigured:135(76845ns) {StartInitialHandshake:157(73528ns) {DropHandshakes:469(1673ns) {DropIncomingHandshake:441(265ns) DropOutgoingHandshake:456(295ns)} GenerateSessionVirtualId:988(845ns) PrepareNewSessionHandshake:476(1486ns) {SwitchToState:178(444ns)}}}}}
2025-11-11T17:18:34.269822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085534:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
2025-11-11T17:18:34.278015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518451675085534:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.322349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518447500640540:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(400469769ns) {Configure:104(400460016ns) {ProcessConfigured:135(54206ns) {StartInitialHandshake:157(51649ns) {DropHandshakes:469(2843ns) {DropIncomingHandshake:441(574ns) DropOutgoingHandshake:456(459ns)} GenerateSessionVirtualId:988(1336ns) PrepareNewSessionHandshake:476(2115ns) {SwitchToState:178(612ns)}}}}}
2025-11-11T17:18:34.332018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857286:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.232685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518450526310850:2092] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.242518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518446231343507:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(404658366ns) {Configure:104(404652909ns) {ProcessConfigured:135(52849ns) {StartInitialHandshake:157(50751ns) {DropHandshakes:469(1350ns) {DropIncomingHandshake:441(264ns) DropOutgoingHandshake:456(315ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:476(2673ns) {SwitchToState:178(793ns)}}}}}
2025-11-11T17:18:34.244930Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518446231343533:2072] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.244950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518446231343533:2072] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.252631Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518446231343533:2072] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:34.358058Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518451795607850:2078] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:34.373885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518447500640519:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(582519850ns) {Configure:104(562497668ns) {ProcessConfigured:135(29990309ns) {StartInitialHandshake:157(19804266ns) {DropHandshakes:469(3883ns) {DropIncomingHandshake:441(488ns) DropOutgoingHandshake:456(410ns)} GenerateSessionVirtualId:988(1105ns) PrepareNewSessionHandshake:476(3712ns) {SwitchToState:178(542ns)}}}}}
2025-11-11T17:18:34.373967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518447500640518:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.374056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:34.374110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518447500640518:2048] [node 1] ICP22 created new session: [3:7571518451795607899:2048]
2025-11-11T17:18:34.374154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518451795607899:2048] [node 1] ICS09 handshake done sender: [3:7571518451795607850:2078] self: [3:7571518451795607887:0] peer: [1:7571518445947693639:0] socket: 130 qp: -1
2025-11-11T17:18:34.374190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518451795607899:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:34.374221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:34.375098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518451795607896:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34814
2025-11-11T17:18:34.329646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518451202505411:2074] [node 6] ICP01 ready to work
2025-11-11T17:18:34.270222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518446231343505:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(423889551ns) {Configure:104(423884522ns) {ProcessConfigured:135(28227108ns) {StartInitialHandshake:157(28224433ns) {DropHandshakes:469(3403ns) {DropIncomingHandshake:441(948ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:476(3163ns) {SwitchToState:178(1096ns)}}}}}
2025-11-11T17:18:34.334570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518451202505419:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50484
2025-11-11T17:18:34.341444Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505416:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.341467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518451202505416:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.342944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518451202505416:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:34.343118Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505419:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.410489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518446907538066:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(464517526ns) {Configure:104(464509171ns) {ProcessConfigured:135(15792420ns) {StartInitialHandshake:157(15789544ns) {DropHandshakes:469(3377ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(554ns)} GenerateSessionVirtualId:988(657ns) PrepareNewSessionHandshake:476(1908ns) {SwitchToState:178(985ns)}}}}}
2025-11-11T17:18:34.297744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518447380118179:2059] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13023) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.274843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518446231343518:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.423321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518447500640543:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(523589701ns) {Configure:104(523582973ns) {ProcessConfigured:135(76813ns) {StartInitialHandshake:157(73950ns) {DropHandshakes:469(3486ns) {DropIncomingHandshake:441(940ns) DropOutgoingHandshake:456(548ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:476(2328ns) {SwitchToState:178(676ns)}}}}}
2025-11-11T17:18:34.426382Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607893:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.426412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518451795607893:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.437815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518451795607893:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:34.332570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857287:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.274906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:34.274951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518446231343518:2059] [node 6] ICP22 created new session: [5:7571518450526310881:2059]
2025-11-11T17:18:34.274997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518450526310881:2059] [node 6] ICS09 handshake done sender: [5:7571518446231343533:2072] self: [5:7571518446231343532:0] peer: [6:7571518452584857285:0] socket: 132 qp: -1
2025-11-11T17:18:34.275039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518450526310881:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:34.275074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:34.337720Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518450526310849:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.345840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518446231343518:2059] [node 6] ICP18 handshake (actor [5:7571518450526310849:2091]) is too old
2025-11-11T17:18:34.382160Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518450526310851:2093] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.416251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518446231343507:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.447013Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607853:2080] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.447061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518451795607853:2080] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.450311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518451202505411:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.297767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13023) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:34.415458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518445947693637:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(466609212ns) {Configure:104(466601469ns) {ProcessConfigured:135(15721214ns) {StartInitialHandshake:157(15717131ns) {DropHandshakes:469(1514ns) {DropIncomingHandshake:441(363ns) DropOutgoingHandshake:456(292ns)} GenerateSessionVirtualId:988(1295ns) PrepareNewSessionHandshake:476(3495ns) {SwitchToState:178(1421ns)}}}}}
2025-11-11T17:18:34.454184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:34.454247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518451202505411:2074] [node 6] ICP22 created new session: [2:7571518451202505427:2074]
2025-11-11T17:18:34.454303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518451202505427:2074] [node 6] ICS09 handshake done sender: [2:7571518451202505416:2113] self: [2:7571518451202505415:0] peer: [6:7571518452584857290:0] socket: 155 qp: -1
2025-11-11T17:18:34.454336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518451202505427:2074] [node 6] ICS10 traffic start
2025-11-11T17:18:34.454369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.455907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518451202505425:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50484
2025-11-11T17:18:34.456066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518451202505426:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50484
No RDMA context found for GID: 2025-11-11T17:18:34.333016Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518452584857284:2115] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.450545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518446231343507:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(33985015ns) {IncomingHandshake:261(6626092ns) {DropIncomingHandshake:441(1282ns) PrepareNewSessionHandshake:476(2872ns) {SwitchToState:178(1142ns)} GenerateSessionVirtualId:988(2016ns) IssueIncomingHandshakeReply:193(57126ns) {SwitchToState:178(825ns)}}}
2025-11-11T17:18:34.469878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518446907538046:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(525599928ns) {Configure:104(525589052ns) {ProcessConfigured:135(28035539ns) {StartInitialHandshake:157(28032443ns) {DropHandshakes:469(3436ns) {DropIncomingHandshake:441(740ns) DropOutgoingHandshake:456(679ns)} GenerateSessionVirtualId:988(1143ns) PrepareNewSessionHandshake:476(4142ns) {SwitchToState:178(1629ns)}}}}}
2025-11-11T17:18:34.318473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518447380118167:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(372402891ns) {Configure:104(372398951ns) {ProcessConfigured:135(85294ns) {StartInitialHandshake:157(82434ns) {DropHandshakes:469(3714ns) {DropIncomingHandshake:441(748ns) DropOutgoingHandshake:456(1078ns)} GenerateSessionVirtualId:988(895ns) PrepareNewSessionHandshake:476(2177ns) {SwitchToState:178(819ns)}}}}}
2025-11-11T17:18:34.333270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518448289889928:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.477695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607896:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.415714Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.483633Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505425:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.482895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310887:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.333324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.428074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518445947693610:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.481674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518447500640540:2059] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.321660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518451675085536:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.511125Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505413:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.481778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:34.523695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518446231343506:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(527866511ns) {Configure:104(527859598ns) {ProcessConfigured:135(19933589ns) {StartInitialHandshake:157(19929370ns) {DropHandshakes:469(3762ns) {DropIncomingHandshake:441(1086ns) DropOutgoingHandshake:456(643ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(4524ns) {SwitchToState:178(2473ns)}}}}}
2025-11-11T17:18:34.428121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:34.333372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518448289889928:2062] [node 5] ICP22 created new session: [6:7571518452584857288:2062]
2025-11-11T17:18:34.321731Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518451675085535:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.428171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518445947693610:2049] [node 3] ICP22 created new session: [1:7571518450242660985:2049]
2025-11-11T17:18:34.511158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518451202505413:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.481836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518447500640540:2059] [node 4] ICP22 created new session: [3:7571518451795607906:2059]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.333418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518452584857288:2062] [node 5] ICS09 handshake done sender: [6:7571518452584857284:2115] self: [6:7571518452584857285:0] peer: [5:7571518446231343532:0] socket: 140 qp: -1
2025-11-11T17:18:34.530371Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518450526310851:2093] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:34.523385Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518451202505413:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:34.553723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518446907538047:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.553788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:34.553830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518446907538047:2049] [node 3] ICP22 created new session: [2:7571518451202505433:2049]
2025-11-11T17:18:34.553878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518451202505433:2049] [node 3] ICS09 handshake done sender: [2:7571518451202505413:2112] self: [2:7571518451202505412:0] peer: [3:7571518451795607909:0] socket: 149 qp: -1
2025-11-11T17:18:34.553909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518451202505433:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:34.553944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:34.565046Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505426:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.574799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518446907538046:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.574959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518446907538081:2079] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.574980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518446907538081:2079] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.582801Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518446907538081:2079] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:34.604514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518451202505438:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50484
2025-11-11T17:18:34.604719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518451202505439:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50484
2025-11-11T17:18:34.605676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518446907538079:2059] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.611215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:34.614385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518446907538079:2059] [node 5] ICP22 created new session: [2:7571518451202505440:2059]
2025-11-11T17:18:34.343178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085541:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.481909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518451795607906:2059] [node 4] ICS09 handshake done sender: [3:7571518451795607893:2116] self: [3:7571518451795607892:0] peer: [4:7571518451675085545:0] socket: 152 qp: -1
2025-11-11T17:18:34.428215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518450242660985:2049] [node 3] ICS09 handshake done sender: [1:7571518445947693640:2076] self: [1:7571518445947693639:0] peer: [3:7571518451795607887:0] socket: 119 qp: -1
2025-11-11T17:18:34.650252Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:34.652071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518446907538046:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518451202505426:2118] held: no
2025-11-11T17:18:34.652193Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518451202505426:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:34.652732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518446907538046:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.652798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:34.652851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518446907538046:2048] [node 1] ICP22 created new session: [2:7571518451202505441:2048]
2025-11-11T17:18:34.652896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518451202505441:2048] [node 1] ICS09 handshake done sender: [2:7571518451202505426:2118] self: [2:7571518451202505436:0] peer: [1:7571518450242660977:0] socket: 164 qp: -1
2025-11-11T17:18:34.652924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518451202505441:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:34.652953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:34.533679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518446231343507:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.481945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518451795607906:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:34.333448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518452584857288:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:34.660948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518451202505440:2059] [node 5] ICS09 handshake done sender: [2:7571518446907538081:2079] self: [2:7571518446907538080:0] peer: [5:7571518450526310890:0] socket: 131 qp: -1
2025-11-11T17:18:34.369799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518447380118179:2059] [node 6] ICP35 event processing took too much time PendingConnection:278(79891041ns) {HandleHandshakeStatus:395(79887148ns) {LogHandshakeFail:472(10624ns) {UpdateErrorStateLog:544(4875ns)} DropOutgoingHandshake:456(2001ns) ProcessPendingSessionEvents:495(11605443ns) {DropSessionEvent:512(19072ns) DropSessionEvent:512(21818ns) DropSessionEvent:512(5002ns) DropSessionEvent:512(5408ns) DropSessionEvent:512(4766ns) DropSessionEvent:512(4160ns) DropSessionEvent:512(4578ns)} TransitToErrorState:752(89840ns) {SwitchToState:178(6163ns) ProcessPendingSessionEvents:495(674ns) DropHandshakes:469(9758ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(559ns)}}}}
2025-11-11T17:18:34.414072Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518451675085541:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.447177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518447380118168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.486753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518447380118168:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(29437343ns) {IncomingHandshake:261(29434656ns) {DropIncomingHandshake:441(417ns) PrepareNewSessionHandshake:476(1655ns) {SwitchToState:178(496ns)} GenerateSessionVirtualId:988(347ns) IssueIncomingHandshakeReply:193(29412337ns) {SwitchToState:178(1403ns)}}}
2025-11-11T17:18:34.513886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085546:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
2025-11-11T17:18:34.534678Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:34.535851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518447380118168:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518451675085541:2116] held: no
2025-11-11T17:18:34.555681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518451675085541:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:34.577897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085548:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
2025-11-11T17:18:34.591539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085549:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
2025-11-11T17:18:34.593835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518447380118168:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.593927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:34.593974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518447380118168:2050] [node 3] ICP22 created new session: [4:7571518451675085550:2050]
2025-11-11T17:18:34.594010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518451675085550:2050] [node 3] ICS09 handshake done sender: [4:7571518451675085541:2116] self: [4:7571518451675085545:0] peer: [3:7571518451795607892:0] socket: 153 qp: -1
2025-11-11T17:18:34.594042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518451675085550:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:34.594073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:34.610812Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518451675085549:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.626506Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518451675085548:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.638195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518447380118166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.644359Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518451675085536:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(298398893ns) {Configure:104(298390656ns) {ProcessConfigured:135(32974310ns) {StartInitialHandshake:157(32970798ns) {DropHandshakes:469(2662ns) {DropIncomingHandshake:441(282ns) DropOutgoingHandshake:456(542ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:476(3726ns) {SwitchToState:178(1516ns)}}}}}
2025-11-11T17:18:34.661119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518447380118167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.667712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518451675085543:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.667738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518451675085543:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.680537Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518451675085543:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.428247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518450242660985:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:34.333506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:34.536213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310885:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.699516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518447380118179:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.699623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:34.699670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518447380118179:2059] [node 6] ICP22 created new session: [4:7571518451675085558:2059]
2025-11-11T17:18:34.668736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518451202505440:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:34.705222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518451675085558:2059] [node 6] ICS09 handshake done sender: [4:7571518451675085543:2117] self: [4:7571518451675085542:0] peer: [6:7571518452584857301:0] socket: 116 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.428284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:34.536762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:34.481984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:34.705296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518451675085558:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:34.340977Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518452584857287:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.428551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518445947693610:2049] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.484007Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518451795607853:2080] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:34.668792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:34.341135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518448289889911:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.543813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518446231343507:2050] [node 3] ICP22 created new session: [5:7571518450526310889:2050]
2025-11-11T17:18:34.706832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085560:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
2025-11-11T17:18:34.683629Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505438:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.689809Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518451202505424:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.689842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518451202505424:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.700072Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518451202505424:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:34.700326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518446907538066:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.700386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:34.700432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518446907538066:2061] [node 4] ICP22 created new session: [2:7571518451202505445:2061]
2025-11-11T17:18:34.700475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518451202505445:2061] [node 4] ICS09 handshake done sender: [2:7571518451202505424:2116] self: [2:7571518451202505423:0] peer: [4:7571518451675085556:0] socket: 128 qp: -1
2025-11-11T17:18:34.700507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518451202505445:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:34.700566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:34.573726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518446231343506:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.576730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310892:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.578348Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518450526310885:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.578472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518446231343505:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.589831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518450526310889:2050] [node 3] ICS09 handshake done sender: [5:7571518450526310851:2093] self: [5:7571518450526310884:0] peer: [3:7571518451795607852:0] socket: 138 qp: -1
2025-11-11T17:18:34.589881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518450526310889:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:34.595332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310894:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.601736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:34.605779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518446231343506:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(37287566ns) {IncomingHandshake:261(37286087ns) {DropIncomingHandshake:441(1192ns) PrepareNewSessionHandshake:476(2465ns) {SwitchToState:178(1649ns)} GenerateSessionVirtualId:988(504ns) IssueIncomingHandshakeReply:193(30365691ns) {SwitchToState:178(1722ns)}}}
2025-11-11T17:18:34.610697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518450526310850:2092] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:34.621676Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518450526310885:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:34.622795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518446231343506:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.622850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:34.622901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518446231343506:2049] [node 2] ICP22 created new session: [5:7571518450526310896:2049]
2025-11-11T17:18:34.622973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518450526310896:2049] [node 2] ICS09 handshake done sender: [5:7571518450526310850:2092] self: [5:7571518450526310890:0] peer: [2:7571518446907538080:0] socket: 137 qp: -1
2025-11-11T17:18:34.623016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518450526310896:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:34.623058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:34.651013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518446231343505:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.651072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:34.651121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518446231343505:2048] [node 1] ICP22 created new session: [5:7571518450526310898:2048]
2025-11-11T17:18:34.651161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518450526310898:2048] [node 1] ICS09 handshake done sender: [5:7571518450526310885:2117] self: [5:7571518450526310893:0] peer: [1:7571518450242660979:0] socket: 166 qp: -1
2025-11-11T17:18:34.651196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518450526310898:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:34.651238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:34.664217Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:34.689940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518446231343505:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:34.697731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [5:7571518446231343505:2048] [node 1] ICP35 event processing took too much time StateWork:293(20941440ns) {HandleHandshakeStatus:395(9541854ns) {LogHandshakeFail:472(15247ns) {UpdateErrorStateLog:544(5186ns)}}}
2025-11-11T17:18:34.732789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310900:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.433876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518445947693651:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(467697627ns) {Configure:104(467692944ns) {ProcessConfigured:135(16002557ns) {StartInitialHandshake:157(15999674ns) {DropHandshakes:469(3008ns) {DropIncomingHandshake:441(1269ns) DropOutgoingHandshake:456(179ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:476(3776ns) {SwitchToState:178(1685ns)}}}}}
2025-11-11T17:18:34.707216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518451675085561:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23790
2025-11-11T17:18:34.497997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518447500640519:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.434158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518450242660981:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35702
2025-11-11T17:18:34.434347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518450242660984:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35702
2025-11-11T17:18:34.441118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518445947693609:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(571459697ns) {Configure:104(560503695ns) {ProcessConfigured:135(24630317ns) {StartInitialHandshake:157(16663312ns) {DropHandshakes:469(4616ns) {DropIncomingHandshake:441(732ns) DropOutgoingHandshake:456(416ns)} GenerateSessionVirtualId:988(2474ns) PrepareNewSessionHandshake:476(4538ns) {SwitchToState:178(2146ns)}}}}}
2025-11-11T17:18:34.449946Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518450242660981:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.503409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518445947693646:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(487949740ns) {Configure:104(487942380ns) {ProcessConfigured:135(36013604ns) {StartInitialHandshake:157(36006444ns) {DropHandshakes:469(3524687ns) {DropIncomingHandshake:441(790ns) DropOutgoingHandshake:456(2412ns)} GenerateSessionVirtualId:988(2092ns) PrepareNewSessionHandshake:476(3459ns) {SwitchToState:178(1337ns)}}}}}
2025-11-11T17:18:34.509690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518450242660984:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.542342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518450242660995:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35702
2025-11-11T17:18:34.576892Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518450242660978:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.576914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518450242660978:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.578709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518450242660980:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.578724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518450242660980:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.579626Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518450242660980:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:34.579877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518445947693637:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.579938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:34.579979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518445947693637:2065] [node 5] ICP22 created new session: [1:7571518450242660998:2065]
2025-11-11T17:18:34.580016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518450242660998:2065] [node 5] ICS09 handshake done sender: [1:7571518450242660980:2114] self: [1:7571518450242660979:0] peer: [5:7571518450526310893:0] socket: 117 qp: -1
2025-11-11T17:18:34.580051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518450242660998:2065] [node 5] ICS10 traffic start
2025-11-11T17:18:34.580116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:34.596215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518450242660995:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.605063Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518450242660978:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:34.617733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518450242660987:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.617768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518450242660987:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.617884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518445947693609:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.633671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:34.640483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518445947693609:2048] [node 2] ICP22 created new session: [1:7571518450242661002:2048]
2025-11-11T17:18:34.645672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518450242661002:2048] [node 2] ICS09 handshake done sender: [1:7571518450242660978:2113] self: [1:7571518450242660977:0] peer: [2:7571518451202505436:0] socket: 162 qp: -1
2025-11-11T17:18:34.649828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518450242660987:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:34.658109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518445947693651:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.658172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:34.658214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518445947693651:2063] [node 6] ICP22 created new session: [1:7571518450242661004:2063]
2025-11-11T17:18:34.658257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518450242661004:2063] [node 6] ICS09 handshake done sender: [1:7571518450242660987:2117] self: [1:7571518450242660983:0] peer: [6:7571518452584857299:0] socket: 168 qp: -1
2025-11-11T17:18:34.658295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518450242661004:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:34.658329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:34.668835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518450242661002:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:34.668860Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518450242660990:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.668884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518450242660990:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.683767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:34.697840Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518450242660990:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:34.498293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518451795607908:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34814
2025-11-11T17:18:34.718713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518445947693609:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(100602165ns) {HandleHandshakeStatus:329(100598551ns) {DropIncomingHandshake:441(24182ns) DropOutgoingHandshake:456(575ns) SwitchToState:178(824ns) ProcessPendingSessionEvents:495(8377558ns) {ForwardSessionEventToSession:612(8320156ns) ForwardSessionEventToSession:612(8757ns) ForwardSessionEventToSession:612(2403ns) ForwardSessionEventToSession:612(3625ns) ForwardSessionEventToSession:612(8730ns) ForwardSessionEventToSession:612(5193ns) ForwardSessionEventToSession:612(5586ns) ForwardSessionEventToSession:612(3008ns) ForwardSessionEventToSession:612(5163ns) ForwardSessionEventToSession:612(4533ns)}}}
2025-11-11T17:18:34.729955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518445947693646:2062] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.514025Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607908:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.730029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:34.530769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518451795607912:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34814
2025-11-11T17:18:34.730075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518445947693646:2062] [node 4] ICP22 created new session: [1:7571518450242661008:2062]
2025-11-11T17:18:34.541982Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518451795607896:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:34.730116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518450242661008:2062] [node 4] ICS09 handshake done sender: [1:7571518450242660990:2118] self: [1:7571518450242660982:0] peer: [4:7571518451675085553:0] socket: 179 qp: -1
2025-11-11T17:18:34.549835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518447500640545:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.730151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518450242661008:2062] [node 4] ICS10 traffic start
2025-11-11T17:18:34.549997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518447500640519:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.730199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:34.550047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:34.550089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518447500640519:2049] [node 2] ICP22 created new session: [3:7571518451795607913:2049]
2025-11-11T17:18:34.550141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518451795607913:2049] [node 2] ICS09 handshake done sender: [3:7571518451795607896:2117] self: [3:7571518451795607909:0] peer: [2:7571518451202505412:0] socket: 154 qp: -1
2025-11-11T17:18:34.550184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518451795607913:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:34.550219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:34.561848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:34.561906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518447500640545:2064] [node 5] ICP22 created new session: [3:7571518451795607915:2064]
2025-11-11T17:18:34.601797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518451795607915:2064] [node 5] ICS09 handshake done sender: [3:7571518451795607853:2080] self: [3:7571518451795607852:0] peer: [5:7571518450526310884:0] socket: 134 qp: -1
2025-11-11T17:18:34.605787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518451795607915:2064] [node 5] ICS10 traffic start
2025-11-11T17:18:34.605845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:34.640329Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518451795607903:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.640368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518451795607903:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.659254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518447500640545:2064] [node 5] ICP35 event processing took too much time PendingConnection:278(127548555ns) {HandleHandshakeStatus:329(127545107ns) {DropIncomingHandshake:441(31739ns) DropOutgoingHandshake:456(627ns) SwitchToState:178(600ns) ProcessPendingSessionEvents:495(17291445ns) {ForwardSessionEventToSession:612(17245836ns) ForwardSessionEventToSession:612(3933ns) ForwardSessionEventToSession:612(6426ns) ForwardSessionEventToSession:612(2163ns) ForwardSessionEventToSession:612(4855ns) ForwardSessionEventToSession:612(4195ns) ForwardSessionEventToSession:612(2971ns) ForwardSessionEventToSession:612(2481ns) ForwardSessionEventToSession:612(5135ns) ForwardSessionEventToSession:612(2146ns)}}}
2025-11-11T17:18:34.678779Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518451795607903:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:34.679033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518447500640543:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.679114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:34.679163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518447500640543:2068] [node 6] ICP22 created new session: [3:7571518451795607918:2068]
2025-11-11T17:18:34.679211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518451795607918:2068] [node 6] ICS09 handshake done sender: [3:7571518451795607903:2119] self: [3:7571518451795607902:0] peer: [6:7571518452584857300:0] socket: 172 qp: -1
2025-11-11T17:18:34.679245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518451795607918:2068] [node 6] ICS10 traffic start
2025-11-11T17:18:34.679281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:34.761845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518446231343507:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(176370193ns) {HandleHandshakeStatus:329(176367867ns) {DropIncomingHandshake:441(1201ns) DropOutgoingHandshake:456(5812ns) SwitchToState:178(594ns) ProcessPendingSessionEvents:495(71892638ns) {ForwardSessionEventToSession:612(42233727ns) ForwardSessionEventToSession:612(5074ns) ForwardSessionEventToSession:612(14249688ns) ForwardSessionEventToSession:612(11654ns) ForwardSessionEventToSession:612(2741ns) ForwardSessionEventToSession:612(7142965ns) ForwardSessionEventToSession:612(3482ns) ForwardSessionEventToSession:612(2353ns) ForwardSessionEventToSession:612(8217980ns) ForwardSessionEventToSession:612(3225ns)}}}
2025-11-11T17:18:34.767453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518446907538079:2059] [node 5] ICP35 event processing took too much time PendingConnection:278(171087773ns) {HandleHandshakeStatus:329(171084730ns) {DropIncomingHandshake:441(1408ns) DropOutgoingHandshake:456(865ns) SwitchToState:178(1026ns) ProcessPendingSessionEvents:495(42506432ns) {ForwardSessionEventToSession:612(17964988ns) ForwardSessionEventToSession:612(6223ns) ForwardSessionEventToSession:612(11012176ns) ForwardSessionEventToSession:612(8439ns) ForwardSessionEventToSession:612(7947099ns) ForwardSessionEventToSession:612(5176572ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(2340ns) ForwardSessionEventToSession:612(349635ns) ForwardSessionEventToSession:612(3141ns)}}}
2025-11-11T17:18:34.767777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518446907538079:2059] [node 5] ICP18 handshake (actor [2:7571518451202505438:2119]) is too old
2025-11-11T17:18:34.707524Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518451675085549:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:34.707639Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518451675085548:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:34.707936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518447380118166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.707975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518447380118167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.708019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:34.708057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518447380118166:2048] [node 1] ICP22 created new session: [4:7571518451675085562:2048]
2025-11-11T17:18:34.708108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518451675085562:2048] [node 1] ICS09 handshake done sender: [4:7571518451675085549:2120] self: [4:7571518451675085553:0] peer: [1:7571518450242660982:0] socket: 159 qp: -1
2025-11-11T17:18:34.708137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518451675085562:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:34.708176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:34.709892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:34.406181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857291:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.713744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:34.713800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518447380118167:2049] [node 2] ICP22 created new session: [4:7571518451675085564:2049]
2025-11-11T17:18:34.427816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518452584857287:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:34.713843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518451675085564:2049] [node 2] ICS09 handshake done sender: [4:7571518451675085548:2119] self: [4:7571518451675085556:0] peer: [2:7571518451202505423:0] socket: 176 qp: -1
2025-11-11T17:18:34.447224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518448289889911:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.713876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518451675085564:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:34.713917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:34.742202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518447380118179:2059] [node 6] ICP35 event processing took too much time PendingConnection:278(42562227ns) {HandleHandshakeStatus:329(42559000ns) {DropIncomingHandshake:441(20523ns) DropOutgoingHandshake:456(541ns) SwitchToState:178(493ns) ProcessPendingSessionEvents:495(8268093ns) {ForwardSessionEventToSession:612(8262329ns)}}}
2025-11-11T17:18:34.447294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:34.473934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518448289889911:2049] [node 2] ICP22 created new session: [6:7571518452584857293:2049]
2025-11-11T17:18:34.503973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857295:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.512251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518452584857293:2049] [node 2] ICS09 handshake done sender: [6:7571518452584857287:2117] self: [6:7571518452584857290:0] peer: [2:7571518451202505415:0] socket: 156 qp: -1
2025-11-11T17:18:34.522005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518452584857293:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:34.529758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:34.536571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518448289889928:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(120584644ns) {HandleHandshakeStatus:329(120581755ns) {DropIncomingHandshake:441(1003ns) DropOutgoingHandshake:456(10639ns) SwitchToState:178(685ns) ProcessPendingSessionEvents:495(76041655ns) {ForwardSessionEventToSession:612(32856317ns) ForwardSessionEventToSession:612(5444868ns) ForwardSessionEventToSession:612(3652ns) ForwardSessionEventToSession:612(2466ns) ForwardSessionEventToSession:612(21583325ns) ForwardSessionEventToSession:612(3015ns) ForwardSessionEventToSession:612(3955430ns) ForwardSessionEventToSession:612(4761ns) ForwardSessionEventToSession:612(8111887ns) ForwardSessionEventToSession:612(4045069ns)}}}
2025-11-11T17:18:34.570140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518448289889911:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(140203466ns) {HandleHandshakeStatus:329(140200933ns) {DropIncomingHandshake:441(2209ns) DropOutgoingHandshake:456(15367ns) SwitchToState:178(142ns) ProcessPendingSessionEvents:495(150681ns) {ForwardSessionEventToSession:612(76044ns) ForwardSessionEventToSession:612(8201ns) ForwardSessionEventToSession:612(2899ns) ForwardSessionEventToSession:612(27963ns) ForwardSessionEventToSession:612(3303ns) ForwardSessionEventToSession:612(1879ns) ForwardSessionEventToSession:612(11334ns) ForwardSessionEventToSession:612(2458ns) ForwardSessionEventToSession:612(1554ns) ForwardSessionEventToSession:612(4203ns)}}}
2025-11-11T17:18:34.583772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857297:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.601978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518452584857295:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.602179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518448289889910:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.606931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518452584857297:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.607073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518448289889912:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.610095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857298:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.621888Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518452584857298:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.635307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857302:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.645059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518448289889929:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:34.688886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857303:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.689358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518452584857304:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57138
2025-11-11T17:18:34.693655Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:34.694430Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518452584857297:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:34.697699Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:34.701920Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518452584857295:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:34.705530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518448289889912:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518452584857297:2120] held: no
2025-11-11T17:18:34.705677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518448289889912:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.705750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:34.705792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518448289889912:2050] [node 3] ICP22 created new session: [6:7571518452584857305:2050]
2025-11-11T17:18:34.705829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518452584857305:2050] [node 3] ICS09 handshake done sender: [6:7571518452584857297:2120] self: [6:7571518452584857300:0] peer: [3:7571518451795607902:0] socket: 174 qp: -1
2025-11-11T17:18:34.705857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518452584857305:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:34.705890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:34.706296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518448289889910:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518452584857295:2119] held: no
2025-11-11T17:18:34.717694Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518452584857298:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:34.725090Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61020) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:34.731115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518448289889910:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.731176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:34.731216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518448289889910:2048] [node 1] ICP22 created new session: [6:7571518452584857307:2048]
2025-11-11T17:18:34.731259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518452584857307:2048] [node 1] ICS09 handshake done sender: [6:7571518452584857295:2119] self: [6:7571518452584857299:0] peer: [1:7571518450242660983:0] socket: 169 qp: -1
2025-11-11T17:18:34.731287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518452584857307:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:34.731327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:34.731664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518448289889929:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61020) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518452584857298:2121] held: no
2025-11-11T17:18:34.732092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518448289889929:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.732125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:34.732162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518448289889929:2064] [node 4] ICP22 created new session: [6:7571518452584857309:2064]
2025-11-11T17:18:34.732202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518452584857309:2064] [node 4] ICS09 handshake done sender: [6:7571518452584857298:2121] self: [6:7571518452584857301:0] peer: [4:7571518451675085542:0] socket: 147 qp: -1
2025-11-11T17:18:34.732232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518452584857309:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:34.732255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:34.772311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518451675085554:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.772348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518451675085554:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:34.770569Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518450526310900:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:34.770737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518446231343531:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:34.778104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518450526310902:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1844
2025-11-11T17:18:34.782040Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518451675085554:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:34.782985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518451675085536:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:34.783059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:34.783104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518451675085536:2063] [node 5] ICP22 created new session: [4:7571518451675085568:2063]
2025-11-11T17:18:34.783145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518451675085568:2063] [node 5] ICS09 handshake done sender: [4:7571518451675085554:2121] self: [4:7571518451675085552:0] peer: [5:7571518450526310901:0] socket: 118 qp: -1
2025-11-11T17:18:34.783179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518451675085568:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:34.783225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:34.782653Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518450526310900:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:34.784429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518446231343531:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:34.784479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:34.784520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518446231343531:2064] [node 4] ICP22 created new session: [5:7571518450526310903:2064]
2025-11-11T17:18:34.784559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518450526310903:2064] [node 4] ICS09 handshake done sender: [5:7571518450526310900:2122] self: [5:7571518450526310901:0] peer: [4:7571518451675085552:0] socket: 125 qp: -1
2025-11-11T17:18:34.784588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518450526310903:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:34.784625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:34.839265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:34.839491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518451202505441:2048] [node 1] ICS01 socket: 164 reason# EndOfStream
2025-11-11T17:18:34.839553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518446907538046:2048] [node 1] ICP30 unregister session Session# [2:7571518451202505441:2048] VirtualId# [2:7571518451202505436:0]
2025-11-11T17:18:34.839587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518451202505441:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.839672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:34.839708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:34.843700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:34.843945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518451795607899:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-11T17:18:34.843976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518447500640518:2048] [node 1] ICP30 unregister session Session# [3:7571518451795607899:2048] VirtualId# [3:7571518451795607887:0]
2025-11-11T17:18:34.844009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518451795607899:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.844084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:34.844121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:34.846091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:34.846302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518451675085562:2048] [node 1] ICS01 socket: 159 reason# EndOfStream
2025-11-11T17:18:34.846334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518447380118166:2048] [node 1] ICP30 unregister session Session# [4:7571518451675085562:2048] VirtualId# [4:7571518451675085553:0]
2025-11-11T17:18:34.846361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518451675085562:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.846417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:34.846448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:34.848347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:34.848554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518452584857307:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-11-11T17:18:34.848587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518448289889910:2048] [node 1] ICP30 unregister session Session# [6:7571518452584857307:2048] VirtualId# [6:7571518452584857299:0]
2025-11-11T17:18:34.848612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518452584857307:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.848664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:34.848710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:34.849766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:34.850001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518450526310898:2048] [node 1] ICS01 socket: 166 reason# EndOfStream
2025-11-11T17:18:34.850048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518446231343505:2048] [node 1] ICP30 unregister session Session# [5:7571518450526310898:2048] VirtualId# [5:7571518450526310893:0]
2025-11-11T17:18:34.850088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518450526310898:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.850180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:34.850245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:34.941833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:34.942095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518451795607913:2049] [node 2] ICS01 socket: 154 reason# EndOfStream
2025-11-11T17:18:34.942135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518447500640519:2049] [node 2] ICP30 unregister session Session# [3:7571518451795607913:2049] VirtualId# [3:7571518451795607909:0]
2025-11-11T17:18:34.942165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518451795607913:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.942247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:34.942283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:34.961812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:34.962032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518452584857293:2049] [node 2] ICS01 socket: 156 reason# EndOfStream
2025-11-11T17:18:34.962076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518448289889911:2049] [node 2] ICP30 unregister session Session# [6:7571518452584857293:2049] VirtualId# [6:7571518452584857290:0]
2025-11-11T17:18:34.962106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518452584857293:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.962188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:34.962231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:34.962768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:34.962963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518451675085564:2049] [node 2] ICS01 socket: 176 reason# EndOfStream
2025-11-11T17:18:34.963001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518447380118167:2049] [node 2] ICP30 unregister session Session# [4:7571518451675085564:2049] VirtualId# [4:7571518451675085556:0]
2025-11-11T17:18:34.963027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518451675085564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.963098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:34.963134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:34.962284Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:34.962496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518450526310896:2049] [node 2] ICS01 socket: 137 reason# EndOfStream
2025-11-11T17:18:34.962528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518446231343506:2049] [node 2] ICP30 unregister session Session# [5:7571518450526310896:2049] VirtualId# [5:7571518450526310890:0]
2025-11-11T17:18:34.962550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518450526310896:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:34.962632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:34.962691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:35.075239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:35.075453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518450526310889:2050] [node 3] ICS07 socket disconnect 138 reason# EndOfStream
2025-11-11T17:18:35.075477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518450526310889:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:35.075544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:35.075584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518450526310889:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:35.076447Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Couldn't connect to any resolved address
2025-11-11T17:18:35.076679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518446231343507:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:35.076725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518450526310889:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:35.076754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518446231343507:2050] [node 3] ICP30 unregister session Session# [5:7571518450526310889:2050] VirtualId# [5:7571518450526310884:0]
2025-11-11T17:18:35.076798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:35.076822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518446231343507:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:35.076836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65159) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:35.074656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:35.074891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518451675085550:2050] [node 3] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:18:35.074928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518447380118168:2050] [node 3] ICP30 unregister session Session# [4:7571518451675085550:2050] VirtualId# [4:7571518451675085545:0]
2025-11-11T17:18:35.074963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518451675085550:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:35.075066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:35.075113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:35.077725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:35.077922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518452584857305:2050] [node 3] ICS01 socket: 174 reason# EndOfStream
2025-11-11T17:18:35.077954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518448289889912:2050] [node 3] ICP30 unregister session Session# [6:7571518452584857305:2050] VirtualId# [6:7571518452584857300:0]
2025-11-11T17:18:35.077981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518452584857305:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:35.078058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:35.078099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:35.086677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518446231343507:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:35.153788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:35.154013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518450526310903:2064] [node 4] ICS01 socket: 125 reason# EndOfStream
2025-11-11T17:18:35.154056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518446231343531:2064] [node 4] ICP30 unregister session Session# [5:7571518450526310903:2064] VirtualId# [5:7571518450526310901:0]
2025-11-11T17:18:35.154090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518450526310903:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:35.154212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:35.154264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:35.167698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:35.168045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518452584857309:2064] [node 4] ICS01 socket: 147 reason# EndOfStream
2025-11-11T17:18:35.168102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518448289889929:2064] [node 4] ICP30 unregister session Session# [6:7571518452584857309:2064] VirtualId# [6:7571518452584857301:0]
2025-11-11T17:18:35.168137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518452584857309:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:35.168275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:35.168317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:35.227910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:35.228257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518452584857288:2062] [node 5] ICS07 socket disconnect 140 reason# EndOfStream
2025-11-11T17:18:35.228311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518452584857288:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:35.228494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:35.228541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518452584857288:2062] [node 5] ICS15 start handshake
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:35.433648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518454292996469:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:35.433754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518454292996470:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:35.493865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518455214291795:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:35.493915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518455214291796:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:35.557833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518455029630085:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:35.557878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518455029630086:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:35.585473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518457269499768:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:35.585526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518457269499769:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:35.585551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518457269499770:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:35.625916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518456962773251:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:35.625957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518456962773252:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:35.625980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518456962773253:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:35.953319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518455214291823:2075] [node 5] ICP01 ready to work
2025-11-11T17:18:35.971659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518457269499784:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:36.098302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518456394924060:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:36.098400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518456394924061:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:36.098445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518456394924062:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:36.099158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518456394924082:2059] [node 5] ICP01 ready to work
2025-11-11T17:18:36.099433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518456394924083:2065] [node 4] ICP01 ready to work
2025-11-11T17:18:36.077702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518458587963813:2082] [node 4] ICP01 ready to work
2025-11-11T17:18:36.059373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518461564467096:2067] [node 6] ICP01 ready to work
2025-11-11T17:18:36.177811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518458587963829:2084] [node 6] ICP01 ready to work
2025-11-11T17:18:36.190651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518456394924083:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(76956013ns) {Configure:104(76952796ns) {ProcessConfigured:135(66583ns) {StartInitialHandshake:157(64334ns) {DropHandshakes:469(3229ns) {DropIncomingHandshake:441(799ns) DropOutgoingHandshake:456(654ns)} GenerateSessionVirtualId:988(966ns) PrepareNewSessionHandshake:476(1980ns) {SwitchToState:178(643ns)}}}}}
2025-11-11T17:18:36.168473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518459324597443:2082] [node 6] ICP01 ready to work
2025-11-11T17:18:36.214695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518455214291823:2075] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(261082507ns) {Configure:104(261076578ns) {ProcessConfigured:135(77473ns) {StartInitialHandshake:157(74936ns) {DropHandshakes:469(5592ns) {DropIncomingHandshake:441(719ns) DropOutgoingHandshake:456(544ns)} GenerateSessionVirtualId:988(1082ns) PrepareNewSessionHandshake:476(2253ns) {SwitchToState:178(611ns)}}}}}
2025-11-11T17:18:36.215121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518459509259157:2076] [node 6] ICP01 ready to work
2025-11-11T17:18:36.215521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518459509259158:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.241923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518455214291795:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21158225ns) {Configure:104(21153734ns) {ProcessConfigured:135(20111827ns) {StartInitialHandshake:157(20109041ns) {DropHandshakes:469(994ns) {DropIncomingHandshake:441(157ns) DropOutgoingHandshake:456(148ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:476(4387ns) {SwitchToState:178(2837ns)}}}}}
2025-11-11T17:18:36.243760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518459509259159:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55070
2025-11-11T17:18:36.237865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518459324597445:2084] [node 4] ICP01 ready to work
2025-11-11T17:18:36.265973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518461564467096:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(204233915ns) {Configure:104(204231057ns) {ProcessConfigured:135(73129ns) {StartInitialHandshake:157(69892ns) {DropHandshakes:469(3069ns) {DropIncomingHandshake:441(732ns) DropOutgoingHandshake:456(629ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:476(2384ns) {SwitchToState:178(975ns)}}}}}
2025-11-11T17:18:36.274041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518459324597446:2085] [node 5] ICP01 ready to work
2025-11-11T17:18:36.253734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518458587963830:2085] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.266352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518456962773252:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(257531798ns) {Configure:104(257526433ns) {ProcessConfigured:135(55155ns) {StartInitialHandshake:157(53405ns) {DropHandshakes:469(1367ns) {DropIncomingHandshake:441(324ns) DropOutgoingHandshake:456(215ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:476(895ns) {SwitchToState:178(326ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.303298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518458587963837:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52271
2025-11-11T17:18:36.303381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518458587963838:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52271
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.322321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518459509259171:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55070
2025-11-11T17:18:36.322383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518459509259172:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55070
2025-11-11T17:18:36.268715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518461257740616:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:36.268961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518461257740617:2067] [node 4] ICP01 ready to work
2025-11-11T17:18:36.316527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740618:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.316738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740619:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.326213Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518461257740619:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.337946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518456962773252:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.345783Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259156:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.345822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518459509259156:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.346369Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259172:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.347124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740628:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.347342Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518459509259156:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:36.347877Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518461257740619:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:36.348056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518456962773252:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.348112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:36.348149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518456962773252:2049] [node 2] ICP22 created new session: [5:7571518461257740629:2049]
2025-11-11T17:18:36.348189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518461257740629:2049] [node 2] ICS09 handshake done sender: [5:7571518461257740619:2114] self: [5:7571518461257740627:0] peer: [2:7571518459509259155:0] socket: 135 qp: -1
2025-11-11T17:18:36.348216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518461257740629:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:36.348245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:36.345108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518459324597443:2082] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(70573983ns) {Configure:104(70568958ns) {ProcessConfigured:135(76053ns) {StartInitialHandshake:157(72861ns) {DropHandshakes:469(3789ns) {DropIncomingHandshake:441(763ns) DropOutgoingHandshake:456(471ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:476(2186ns) {SwitchToState:178(606ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.366836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891434:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.359186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518458587963846:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52271
2025-11-11T17:18:36.349650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518455214291823:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.367067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891435:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.377966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963846:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.361730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:36.370607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518455214291823:2075] [node 5] ICP22 created new session: [2:7571518459509259175:2075]
2025-11-11T17:18:36.397769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518459509259175:2075] [node 5] ICS09 handshake done sender: [2:7571518459509259156:2110] self: [2:7571518459509259155:0] peer: [5:7571518461257740627:0] socket: 131 qp: -1
2025-11-11T17:18:36.397831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518459509259175:2075] [node 5] ICS10 traffic start
2025-11-11T17:18:36.397883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:36.398342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518455214291823:2075] [node 5] ICP35 event processing took too much time PendingConnection:278(48505363ns) {HandleHandshakeStatus:329(48501971ns) {DropIncomingHandshake:441(12808ns) DropOutgoingHandshake:456(7103ns) SwitchToState:178(671ns) ProcessPendingSessionEvents:495(101081ns) {ForwardSessionEventToSession:612(53802ns) ForwardSessionEventToSession:612(3326ns) ForwardSessionEventToSession:612(6744ns) ForwardSessionEventToSession:612(3391ns) ForwardSessionEventToSession:612(5719ns) ForwardSessionEventToSession:612(4777ns) ForwardSessionEventToSession:612(2545ns) ForwardSessionEventToSession:612(4758ns) ForwardSessionEventToSession:612(2706ns) ForwardSessionEventToSession:612(2586ns)}}}
2025-11-11T17:18:36.410226Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259171:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.411435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518455214291795:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.423170Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963832:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.423210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518458587963832:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.423776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963838:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.424259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518458587963848:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52271
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.425316Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518458587963832:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:36.434477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518459324597446:2085] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40932551ns) {Configure:104(40928792ns) {ProcessConfigured:135(63671ns) {StartInitialHandshake:157(60506ns) {DropHandshakes:469(3082ns) {DropIncomingHandshake:441(619ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(1340ns) PrepareNewSessionHandshake:476(2161ns) {SwitchToState:178(590ns)}}}}}
2025-11-11T17:18:36.435620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518459324597449:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-11T17:18:36.435774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518459324597450:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-11T17:18:36.435882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518459324597451:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.435970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518459324597452:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-11T17:18:36.426188Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518460689891435:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.425107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518459509259179:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55070
2025-11-11T17:18:36.425672Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518461257740618:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.436057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518459324597453:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-11T17:18:36.446170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518461564467142:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11583
2025-11-11T17:18:36.446472Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963848:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.475355Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597451:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.475729Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259159:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.446552Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963837:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.446704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518454292996469:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.446781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:36.446825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518454292996469:2048] [node 2] ICP22 created new session: [1:7571518458587963851:2048]
2025-11-11T17:18:36.446867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518458587963851:2048] [node 2] ICS09 handshake done sender: [1:7571518458587963832:2110] self: [1:7571518458587963831:0] peer: [2:7571518459509259178:0] socket: 140 qp: -1
2025-11-11T17:18:36.446906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518458587963851:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:36.446949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:36.447578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518454292996469:2048] [node 2] ICP18 handshake (actor [1:7571518458587963848:2118]) is too old
2025-11-11T17:18:36.474222Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963834:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.474246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518458587963834:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.475436Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597452:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.476405Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259161:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.476433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518459509259161:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.475517Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597449:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.475894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518455029630085:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.429982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518460689891434:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.454716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518456394924060:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.454784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518456394924061:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.454968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891436:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.455317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891437:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.477689Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.482112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597450:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.482259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518455029630086:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.488725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518458587963834:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:36.481901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518460689891437:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.488418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891440:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.494463Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963836:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.494498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518458587963836:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.495118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518458587963829:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.495214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:36.495267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518458587963829:2084] [node 6] ICP22 created new session: [1:7571518458587963856:2084]
2025-11-11T17:18:36.495311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518458587963856:2084] [node 6] ICS09 handshake done sender: [1:7571518458587963834:2111] self: [1:7571518458587963833:0] peer: [6:7571518460689891438:0] socket: 141 qp: -1
2025-11-11T17:18:36.495345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518458587963856:2084] [node 6] ICS10 traffic start
2025-11-11T17:18:36.495386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:36.498206Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518459509259171:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:36.493664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597453:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.497721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518461564467143:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11583
2025-11-11T17:18:36.499190Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518461564467142:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.499296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518457269499770:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.498814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518461257740617:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.509966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597457:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.509660Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518458587963836:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.505922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518455214291795:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518459509259171:2116] held: no
2025-11-11T17:18:36.514551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518454292996470:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.514621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:36.514658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518454292996470:2049] [node 3] ICP22 created new session: [1:7571518458587963860:2049]
2025-11-11T17:18:36.514697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518458587963860:2049] [node 3] ICS09 handshake done sender: [1:7571518458587963836:2112] self: [1:7571518458587963835:0] peer: [3:7571518459324597466:0] socket: 142 qp: -1
2025-11-11T17:18:36.514731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518458587963860:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:36.514759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:36.513777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518456394924062:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.510004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518459324597457:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.511445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740633:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.513710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518457269499786:2063] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.513751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518457269499786:2063] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.539237Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518461564467143:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.539313Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:36.511033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518459324597470:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.506035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518455214291795:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.521728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518460689891436:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.537766Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:36.546085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518456394924060:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518460689891435:2116] held: no
2025-11-11T17:18:36.546552Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:36.511965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518459324597471:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-11T17:18:36.512067Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518459324597452:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:36.512357Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518459324597457:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:36.513439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518455029630085:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.513510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:36.513557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518455029630085:2048] [node 1] ICP22 created new session: [3:7571518459324597473:2048]
2025-11-11T17:18:36.513660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518459324597473:2048] [node 1] ICS09 handshake done sender: [3:7571518459324597452:2114] self: [3:7571518459324597466:0] peer: [1:7571518458587963835:0] socket: 156 qp: -1
2025-11-11T17:18:36.513698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518459324597473:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:36.513738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:36.522810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518459324597445:2084] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.522913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:36.522964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518459324597445:2084] [node 4] ICP22 created new session: [3:7571518459324597475:2084]
2025-11-11T17:18:36.523048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518459324597475:2084] [node 4] ICS09 handshake done sender: [3:7571518459324597457:2117] self: [3:7571518459324597456:0] peer: [4:7571518461564467144:0] socket: 164 qp: -1
2025-11-11T17:18:36.523091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518459324597475:2084] [node 4] ICS10 traffic start
2025-11-11T17:18:36.523132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:36.524731Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518459324597450:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:36.553949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518455029630086:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.554031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:36.554076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518455029630086:2049] [node 2] ICP22 created new session: [3:7571518459324597478:2049]
2025-11-11T17:18:36.554143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518459324597478:2049] [node 2] ICS09 handshake done sender: [3:7571518459324597450:2112] self: [3:7571518459324597468:0] peer: [2:7571518459509259162:0] socket: 154 qp: -1
2025-11-11T17:18:36.554181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518459324597478:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:36.554215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:36.563167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518454292996470:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(25578019ns) {HandleHandshakeStatus:329(25569284ns) {DropIncomingHandshake:441(197ns) DropOutgoingHandshake:456(934ns) SwitchToState:178(458ns) ProcessPendingSessionEvents:495(25267973ns) {ForwardSessionEventToSession:612(38756ns) ForwardSessionEventToSession:612(6133ns) ForwardSessionEventToSession:612(2183ns) ForwardSessionEventToSession:612(2562ns) ForwardSessionEventToSession:612(6995899ns) ForwardSessionEventToSession:612(10231ns) ForwardSessionEventToSession:612(14501ns) ForwardSessionEventToSession:612(2640205ns) ForwardSessionEventToSession:612(2839ns) ForwardSessionEventToSession:612(5960329ns)}}}
2025-11-11T17:18:36.506076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.567982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740634:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.568072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740635:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.569866Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518460689891435:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:36.506112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518455214291795:2048] [node 1] ICP22 created new session: [2:7571518459509259182:2048]
2025-11-11T17:18:36.506152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518459509259182:2048] [node 1] ICS09 handshake done sender: [2:7571518459509259171:2116] self: [2:7571518459509259178:0] peer: [1:7571518458587963831:0] socket: 147 qp: -1
2025-11-11T17:18:36.506184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518459509259182:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:36.506218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:36.506608Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259163:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.506629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518459509259163:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.508433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518459509259180:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55070
2025-11-11T17:18:36.515028Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518459509259161:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:36.521933Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518459509259163:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:36.522308Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259180:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.529836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518459509259157:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.529898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:36.529949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518459509259157:2076] [node 6] ICP22 created new session: [2:7571518459509259187:2076]
2025-11-11T17:18:36.530014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518459509259187:2076] [node 6] ICS09 handshake done sender: [2:7571518459509259161:2112] self: [2:7571518459509259160:0] peer: [6:7571518460689891439:0] socket: 132 qp: -1
2025-11-11T17:18:36.530060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518459509259187:2076] [node 6] ICS10 traffic start
2025-11-11T17:18:36.530101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:36.530563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518455214291796:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.530593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:36.530630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518455214291796:2049] [node 3] ICP22 created new session: [2:7571518459509259189:2049]
2025-11-11T17:18:36.530672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518459509259189:2049] [node 3] ICS09 handshake done sender: [2:7571518459509259163:2113] self: [2:7571518459509259162:0] peer: [3:7571518459324597468:0] socket: 133 qp: -1
2025-11-11T17:18:36.530720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518459509259189:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:36.530750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:36.552141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518455214291796:2049] [node 3] ICP18 handshake (actor [2:7571518459509259180:2119]) is too old
2025-11-11T17:18:36.552985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518457269499770:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518461564467142:2115] held: no
2025-11-11T17:18:36.553481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518457269499768:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.553603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518461564467145:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11583
2025-11-11T17:18:36.553739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518461564467146:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11583
2025-11-11T17:18:36.571005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518457269499786:2063] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:36.571494Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518461564467145:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.571541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518457269499784:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.571576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:36.571617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518457269499784:2062] [node 5] ICP22 created new session: [4:7571518461564467150:2062]
2025-11-11T17:18:36.571667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518461564467150:2062] [node 5] ICS09 handshake done sender: [4:7571518457269499786:2063] self: [4:7571518457269499785:0] peer: [5:7571518461257740632:0] socket: 122 qp: -1
2025-11-11T17:18:36.571694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518461564467150:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:36.571725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:36.572243Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518461564467142:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.574156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518459509259191:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55070
2025-11-11T17:18:36.565551Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597448:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.565611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518459324597448:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.573923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518457269499770:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.573976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:36.574019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518457269499770:2050] [node 3] ICP22 created new session: [4:7571518461564467152:2050]
2025-11-11T17:18:36.574067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518461564467152:2050] [node 3] ICS09 handshake done sender: [4:7571518461564467142:2115] self: [4:7571518461564467144:0] peer: [3:7571518459324597456:0] socket: 165 qp: -1
2025-11-11T17:18:36.574103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518461564467152:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:36.574131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:36.583137Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963840:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.583178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518458587963840:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.589887Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518458587963840:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:36.592428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518457269499769:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.597934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518456394924061:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518460689891434:2115] held: no
2025-11-11T17:18:36.598128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518456394924060:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.598173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:36.598219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518456394924060:2048] [node 1] ICP22 created new session: [6:7571518460689891443:2048]
2025-11-11T17:18:36.598259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518460689891443:2048] [node 1] ICS09 handshake done sender: [6:7571518460689891435:2116] self: [6:7571518460689891438:0] peer: [1:7571518458587963833:0] socket: 152 qp: -1
2025-11-11T17:18:36.598293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518460689891443:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:36.598322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:36.598901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518456394924082:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.599218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891442:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.590164Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518461257740633:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.590234Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518461257740618:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:36.591376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518461257740617:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.593088Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518458587963842:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.607040Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259167:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.607082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518459509259167:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.606594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518461564467156:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11583
2025-11-11T17:18:36.614571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891445:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.614675Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518460689891434:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:36.615112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518456394924061:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.615164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:36.615211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518456394924061:2049] [node 2] ICP22 created new session: [6:7571518460689891449:2049]
2025-11-11T17:18:36.615354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518460689891449:2049] [node 2] ICS09 handshake done sender: [6:7571518460689891434:2115] self: [6:7571518460689891439:0] peer: [2:7571518459509259160:0] socket: 143 qp: -1
2025-11-11T17:18:36.615456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518460689891449:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:36.615492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:36.615532Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518459509259167:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:36.591423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:36.593121Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518458587963842:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.629277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518458587963813:2082] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.629358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:36.629405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518458587963813:2082] [node 4] ICP22 created new session: [1:7571518458587963863:2082]
2025-11-11T17:18:36.629452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518458587963863:2082] [node 4] ICS09 handshake done sender: [1:7571518458587963840:2115] self: [1:7571518458587963839:0] peer: [4:7571518461564467148:0] socket: 158 qp: -1
2025-11-11T17:18:36.629490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518458587963863:2082] [node 4] ICS10 traffic start
2025-11-11T17:18:36.629545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:36.591468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518461257740617:2067] [node 4] ICP22 created new session: [5:7571518461257740636:2067]
2025-11-11T17:18:36.591512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518461257740636:2067] [node 4] ICS09 handshake done sender: [5:7571518461257740618:2113] self: [5:7571518461257740632:0] peer: [4:7571518457269499785:0] socket: 134 qp: -1
2025-11-11T17:18:36.591550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518461257740636:2067] [node 4] ICS10 traffic start
2025-11-11T17:18:36.591580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:36.592402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518456962773251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.635456Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518461257740621:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.635517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518461257740621:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.638560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518459509259158:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.638617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:36.638665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518459509259158:2077] [node 4] ICP22 created new session: [2:7571518459509259193:2077]
2025-11-11T17:18:36.638711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518459509259193:2077] [node 4] ICS09 handshake done sender: [2:7571518459509259167:2115] self: [2:7571518459509259166:0] peer: [4:7571518461564467155:0] socket: 175 qp: -1
2025-11-11T17:18:36.638746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518459509259193:2077] [node 4] ICS10 traffic start
2025-11-11T17:18:36.638781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.646125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891451:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.650010Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518459509259191:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.657928Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518459324597448:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:36.667605Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:36.668812Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518461564467143:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:36.669114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518457269499768:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518461564467143:2116] held: no
2025-11-11T17:18:36.669203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518457269499768:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.677425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518459324597443:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.677500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:36.677548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518459324597443:2082] [node 6] ICP22 created new session: [3:7571518459324597481:2082]
2025-11-11T17:18:36.677603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518459324597481:2082] [node 6] ICS09 handshake done sender: [3:7571518459324597448:2110] self: [3:7571518459324597447:0] peer: [6:7571518460689891441:0] socket: 162 qp: -1
2025-11-11T17:18:36.677648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518459324597481:2082] [node 6] ICS10 traffic start
2025-11-11T17:18:36.677686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.676840Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518460689891437:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:36.678793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891452:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.679130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518456394924062:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.679179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:36.679223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518456394924062:2050] [node 3] ICP22 created new session: [6:7571518460689891453:2050]
2025-11-11T17:18:36.679264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518460689891453:2050] [node 3] ICS09 handshake done sender: [6:7571518460689891437:2118] self: [6:7571518460689891441:0] peer: [3:7571518459324597447:0] socket: 166 qp: -1
2025-11-11T17:18:36.679291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518460689891453:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:36.679321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:36.681250Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518460689891436:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:36.681574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518456394924082:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.681671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:36.675837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518459509259158:2077] [node 4] ICP18 handshake (actor [2:7571518459509259191:2120]) is too old
2025-11-11T17:18:36.654551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740639:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.685727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518456394924082:2059] [node 5] ICP22 created new session: [6:7571518460689891455:2059]
2025-11-11T17:18:36.685799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518460689891455:2059] [node 5] ICS09 handshake done sender: [6:7571518460689891436:2117] self: [6:7571518460689891446:0] peer: [5:7571518461257740620:0] socket: 161 qp: -1
2025-11-11T17:18:36.685849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518460689891455:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:36.685892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:36.670123Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518461257740621:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:36.669246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:36.669291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518457269499768:2048] [node 1] ICP22 created new session: [4:7571518461564467159:2048]
2025-11-11T17:18:36.669329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518461564467159:2048] [node 1] ICS09 handshake done sender: [4:7571518461564467143:2116] self: [4:7571518461564467148:0] peer: [1:7571518458587963839:0] socket: 169 qp: -1
2025-11-11T17:18:36.669366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518461564467159:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:36.669400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:36.674179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518461564467157:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11583
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.694365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518461564467158:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11583
2025-11-11T17:18:36.694805Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:36.701375Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518461257740639:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.701487Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518461257740635:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.704227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518457269499769:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518461564467145:2117] held: no
2025-11-11T17:18:36.704420Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518461564467145:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:36.700973Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518458587963842:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:36.703278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518458587963830:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.703348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:36.703392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518458587963830:2085] [node 5] ICP22 created new session: [1:7571518458587963866:2085]
2025-11-11T17:18:36.703431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518458587963866:2085] [node 5] ICS09 handshake done sender: [1:7571518458587963842:2116] self: [1:7571518458587963841:0] peer: [5:7571518461257740638:0] socket: 148 qp: -1
2025-11-11T17:18:36.703467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518458587963866:2085] [node 5] ICS10 traffic start
2025-11-11T17:18:36.703493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:36.701696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518461257740616:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.701745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:36.701801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518461257740616:2064] [node 6] ICP22 created new session: [5:7571518461257740642:2064]
2025-11-11T17:18:36.701841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518461257740642:2064] [node 6] ICS09 handshake done sender: [5:7571518461257740621:2115] self: [5:7571518461257740620:0] peer: [6:7571518460689891446:0] socket: 144 qp: -1
2025-11-11T17:18:36.701881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518461257740642:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:36.701921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:36.702807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518461257740616:2064] [node 6] ICP18 handshake (actor [5:7571518461257740639:2121]) is too old
2025-11-11T17:18:36.702912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740641:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.705307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518457269499769:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.705357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:36.705397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518457269499769:2049] [node 2] ICP22 created new session: [4:7571518461564467161:2049]
2025-11-11T17:18:36.705439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518461564467161:2049] [node 2] ICS09 handshake done sender: [4:7571518461564467145:2117] self: [4:7571518461564467155:0] peer: [2:7571518459509259166:0] socket: 155 qp: -1
2025-11-11T17:18:36.705470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518461564467161:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:36.705500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:36.706878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518460689891445:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.707308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518456394924083:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.715146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518461564467158:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.717910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518459324597461:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.717948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518459324597461:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.719167Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518459324597461:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:36.719462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518459324597446:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.719535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:36.719577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518459324597446:2085] [node 5] ICP22 created new session: [3:7571518459324597484:2085]
2025-11-11T17:18:36.719642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518459324597484:2085] [node 5] ICS09 handshake done sender: [3:7571518459324597461:2119] self: [3:7571518459324597460:0] peer: [5:7571518461257740644:0] socket: 128 qp: -1
2025-11-11T17:18:36.719680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518459324597484:2085] [node 5] ICS10 traffic start
2025-11-11T17:18:36.719723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:36.713894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518456962773253:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:36.716393Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:36.717684Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518461257740633:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.721526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518460689891458:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19316
2025-11-11T17:18:36.730578Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518460689891445:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:36.732131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518456394924083:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.732198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:36.732240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518456394924083:2065] [node 4] ICP22 created new session: [6:7571518460689891459:2065]
2025-11-11T17:18:36.732280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518460689891459:2065] [node 4] ICS09 handshake done sender: [6:7571518460689891445:2121] self: [6:7571518460689891457:0] peer: [4:7571518461564467138:0] socket: 183 qp: -1
2025-11-11T17:18:36.732309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518460689891459:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:36.732351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:36.733751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518456962773251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518461257740633:2118] held: no
2025-11-11T17:18:36.733870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518456962773251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.733910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:36.733951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518456962773251:2048] [node 1] ICP22 created new session: [5:7571518461257740645:2048]
2025-11-11T17:18:36.733992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518461257740645:2048] [node 1] ICS09 handshake done sender: [5:7571518461257740633:2118] self: [5:7571518461257740638:0] peer: [1:7571518458587963841:0] socket: 172 qp: -1
2025-11-11T17:18:36.734023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518461257740645:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:36.715309Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518461564467139:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:36.737291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:36.737938Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:36.715331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518461564467139:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:36.729931Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518461564467139:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:36.731103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518461564467096:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:36.731162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:36.731211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518461564467096:2067] [node 6] ICP22 created new session: [4:7571518461564467165:2067]
2025-11-11T17:18:36.731256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518461564467165:2067] [node 6] ICS09 handshake done sender: [4:7571518461564467139:2114] self: [4:7571518461564467138:0] peer: [6:7571518460689891457:0] socket: 181 qp: -1
2025-11-11T17:18:36.731293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518461564467165:2067] [node 6] ICS10 traffic start
2025-11-11T17:18:36.731343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:36.770055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518456962773253:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518461257740635:2120] held: no
2025-11-11T17:18:36.770422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518461257740647:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56339
2025-11-11T17:18:36.771057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518461257740635:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:36.771306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518456962773253:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:36.771362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:36.771407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518456962773253:2050] [node 3] ICP22 created new session: [5:7571518461257740648:2050]
2025-11-11T17:18:36.771446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518461257740648:2050] [node 3] ICS09 handshake done sender: [5:7571518461257740635:2120] self: [5:7571518461257740644:0] peer: [3:7571518459324597460:0] socket: 168 qp: -1
2025-11-11T17:18:36.771482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518461257740648:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:36.771525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:36.822965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:36.823197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518459509259182:2048] [node 1] ICS01 socket: 147 reason# EndOfStream
2025-11-11T17:18:36.823238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518455214291795:2048] [node 1] ICP30 unregister session Session# [2:7571518459509259182:2048] VirtualId# [2:7571518459509259178:0]
2025-11-11T17:18:36.823269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518459509259182:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.823363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:36.823408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:36.825725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:36.825936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518459324597473:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-11T17:18:36.825978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518455029630085:2048] [node 1] ICP30 unregister session Session# [3:7571518459324597473:2048] VirtualId# [3:7571518459324597466:0]
2025-11-11T17:18:36.826014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518459324597473:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.826125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:36.826178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:36.837663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:36.837921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518461564467159:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-11-11T17:18:36.837984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518457269499768:2048] [node 1] ICP30 unregister session Session# [4:7571518461564467159:2048] VirtualId# [4:7571518461564467148:0]
2025-11-11T17:18:36.838023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518461564467159:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.838133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:36.838182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:36.837755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:36.837998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518460689891443:2048] [node 1] ICS01 socket: 152 reason# EndOfStream
2025-11-11T17:18:36.838033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518456394924060:2048] [node 1] ICP30 unregister session Session# [6:7571518460689891443:2048] VirtualId# [6:7571518460689891438:0]
2025-11-11T17:18:36.838065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518460689891443:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.838154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:36.838192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:36.838329Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:36.869880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518461257740645:2048] [node 1] ICS01 socket: 172 reason# EndOfStream
2025-11-11T17:18:36.869938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518456962773251:2048] [node 1] ICP30 unregister session Session# [5:7571518461257740645:2048] VirtualId# [5:7571518461257740638:0]
2025-11-11T17:18:36.869974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518461257740645:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.870102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:36.870146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:36.896860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:36.897152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518459324597478:2049] [node 2] ICS07 socket disconnect 154 reason# EndOfStream
2025-11-11T17:18:36.897180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518459324597478:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.897317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:36.897368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518459324597478:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:36.902346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:36.902525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518461564467161:2049] [node 2] ICS01 socket: 155 reason# EndOfStream
2025-11-11T17:18:36.902620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518457269499769:2049] [node 2] ICP30 unregister session Session# [4:7571518461564467161:2049] VirtualId# [4:7571518461564467155:0]
2025-11-11T17:18:36.902651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518461564467161:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.902712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:36.903172Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Couldn't connect to any resolved address
2025-11-11T17:18:36.903420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518455029630086:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:36.903470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518459324597478:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:36.903502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518455029630086:2049] [node 2] ICP30 unregister session Session# [3:7571518459324597478:2049] VirtualId# [3:7571518459324597468:0]
2025-11-11T17:18:36.903540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:36.903574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518455029630086:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:36.903588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7895) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:36.902748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:36.901859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:36.902075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518461257740629:2049] [node 2] ICS01 socket: 135 reason# EndOfStream
2025-11-11T17:18:36.902115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518456962773252:2049] [node 2] ICP30 unregister session Session# [5:7571518461257740629:2049] VirtualId# [5:7571518461257740627:0]
2025-11-11T17:18:36.902144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518461257740629:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.902234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:36.902285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:36.905713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:36.906002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518460689891449:2049] [node 2] ICS01 socket: 143 reason# EndOfStream
2025-11-11T17:18:36.906099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518456394924061:2049] [node 2] ICP30 unregister session Session# [6:7571518460689891449:2049] VirtualId# [6:7571518460689891439:0]
2025-11-11T17:18:36.906154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518460689891449:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.906311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:36.906424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:36.920979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518455029630086:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:36.969808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:36.970047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518460689891453:2050] [node 3] ICS01 socket: 166 reason# EndOfStream
2025-11-11T17:18:36.970083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518456394924062:2050] [node 3] ICP30 unregister session Session# [6:7571518460689891453:2050] VirtualId# [6:7571518460689891441:0]
2025-11-11T17:18:36.970118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518460689891453:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.970205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:36.970249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:36.970339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:36.970536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518461564467152:2050] [node 3] ICS01 socket: 165 reason# EndOfStream
2025-11-11T17:18:36.970560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518457269499770:2050] [node 3] ICP30 unregister session Session# [4:7571518461564467152:2050] VirtualId# [4:7571518461564467144:0]
2025-11-11T17:18:36.970580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518461564467152:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.970632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:36.970662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:36.970745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:36.970910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518461257740648:2050] [node 3] ICS01 socket: 168 reason# EndOfStream
2025-11-11T17:18:36.970942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518456962773253:2050] [node 3] ICP30 unregister session Session# [5:7571518461257740648:2050] VirtualId# [5:7571518461257740644:0]
2025-11-11T17:18:36.970961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518461257740648:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:36.971006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:36.971038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:37.017990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:37.018260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518461257740636:2067] [node 4] ICS01 socket: 134 reason# EndOfStream
2025-11-11T17:18:37.018302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518461257740617:2067] [node 4] ICP30 unregister session Session# [5:7571518461257740636:2067] VirtualId# [5:7571518461257740632:0]
2025-11-11T17:18:37.018335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518461257740636:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:37.018439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:37.018491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:37.018603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:37.018778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518460689891459:2065] [node 4] ICS01 socket: 183 reason# EndOfStream
2025-11-11T17:18:37.018811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518456394924083:2065] [node 4] ICP30 unregister session Session# [6:7571518460689891459:2065] VirtualId# [6:7571518460689891457:0]
2025-11-11T17:18:37.018836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518460689891459:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:37.018906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:37.018952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:37.089193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:37.089415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518460689891455:2059] [node 5] ICS01 socket: 161 reason# EndOfStream
2025-11-11T17:18:37.089455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518456394924082:2059] [node 5] ICP30 unregister session Session# [6:7571518460689891455:2059] VirtualId# [6:7571518460689891446:0]
2025-11-11T17:18:37.089488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518460689891455:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:37.089561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:37.089622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:37.277972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518466014337037:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:37.278023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518466014337038:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:37.317897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518466160984475:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:37.317943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518466160984476:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:37.349888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518466059247040:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:37.349925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518466059247041:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:37.397956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518464154090660:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:37.397994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518464154090661:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:37.398017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518464154090662:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:37.425916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518465741180248:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:37.425960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518465741180249:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:37.425986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518465741180250:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:37.626841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518462264007041:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:37.626896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518462264007042:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:37.626920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518462264007043:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:37.781684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518464154090676:2061] [node 5] ICP01 ready to work
2025-11-11T17:18:37.766050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518466160984496:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:37.773903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518466014337064:2071] [node 4] ICP01 ready to work
2025-11-11T17:18:37.817702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518466160984499:2067] [node 4] ICP01 ready to work
2025-11-11T17:18:37.866798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518466059247040:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(161282916ns) {Configure:104(161275448ns) {ProcessConfigured:135(97563ns) {StartInitialHandshake:157(92792ns) {DropHandshakes:469(2075ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(437ns)} GenerateSessionVirtualId:988(925ns) PrepareNewSessionHandshake:476(936ns) {SwitchToState:178(359ns)}}}}}
2025-11-11T17:18:37.873757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518466014337065:2072] [node 6] ICP01 ready to work
2025-11-11T17:18:37.873836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518466059247061:2065] [node 4] ICP01 ready to work
2025-11-11T17:18:37.874010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518466059247064:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:37.905913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518466014337066:2073] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:37.951148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518466014337037:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(245742229ns) {Configure:104(245728130ns) {ProcessConfigured:135(53363ns) {StartInitialHandshake:157(48926ns) {DropHandshakes:469(2570ns) {DropIncomingHandshake:441(549ns) DropOutgoingHandshake:456(409ns)} GenerateSessionVirtualId:988(624ns) PrepareNewSessionHandshake:476(1692ns) {SwitchToState:178(606ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:37.949995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518466160984501:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-11-11T17:18:37.954287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518466160984508:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-11-11T17:18:37.934856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518466059247041:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(56641264ns) {Configure:104(56636959ns) {ProcessConfigured:135(73962ns) {StartInitialHandshake:157(71593ns) {DropHandshakes:469(2443ns) {DropIncomingHandshake:441(412ns) DropOutgoingHandshake:456(607ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:476(1188ns) {SwitchToState:178(421ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:37.955988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518466014337064:2071] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(49744589ns) {Configure:104(49740036ns) {ProcessConfigured:135(69324ns) {StartInitialHandshake:157(68104ns) {DropHandshakes:469(878ns) {DropIncomingHandshake:441(154ns) DropOutgoingHandshake:456(98ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:476(1384ns) {SwitchToState:178(252ns)}}}}}
2025-11-11T17:18:37.956749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518466014337075:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35876
2025-11-11T17:18:37.956803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518466014337076:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35876
2025-11-11T17:18:37.955716Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518466160984501:2071] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:37.959951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518466160984513:2076] [node 5] ICP01 ready to work
2025-11-11T17:18:37.942877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518466059247061:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53013812ns) {Configure:104(53009384ns) {ProcessConfigured:135(52416ns) {StartInitialHandshake:157(50824ns) {DropHandshakes:469(1250ns) {DropIncomingHandshake:441(250ns) DropOutgoingHandshake:456(305ns)} GenerateSessionVirtualId:988(393ns) PrepareNewSessionHandshake:476(798ns) {SwitchToState:178(301ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:37.957614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518464154090660:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(147571076ns) {Configure:104(147565954ns) {ProcessConfigured:135(2536070ns) {StartInitialHandshake:157(2533991ns) {DropHandshakes:469(1254ns) {DropIncomingHandshake:441(322ns) DropOutgoingHandshake:456(211ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:476(1925ns) {SwitchToState:178(143ns)}}}}}
2025-11-11T17:18:37.977424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518466014337082:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35876
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:37.971439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518464154090662:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(145580422ns) {Configure:104(145574815ns) {ProcessConfigured:135(44266ns) {StartInitialHandshake:157(42336ns) {DropHandshakes:469(2844ns) {DropIncomingHandshake:441(440ns) DropOutgoingHandshake:456(653ns)} GenerateSessionVirtualId:988(838ns) PrepareNewSessionHandshake:476(1289ns) {SwitchToState:178(535ns)}}}}}
2025-11-11T17:18:37.978769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518466014337066:2073] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16633050ns) {Configure:104(16628660ns) {ProcessConfigured:135(240290ns) {StartInitialHandshake:157(236154ns) {DropHandshakes:469(3679ns) {DropIncomingHandshake:441(850ns) DropOutgoingHandshake:456(636ns)} GenerateSessionVirtualId:988(1329ns) PrepareNewSessionHandshake:476(3101ns) {SwitchToState:178(1153ns)}}}}}
2025-11-11T17:18:37.993799Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337082:2081] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:37.989817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518466160984499:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(132882043ns) {Configure:104(132870543ns) {ProcessConfigured:135(63719ns) {StartInitialHandshake:157(61393ns) {DropHandshakes:469(3406ns) {DropIncomingHandshake:441(1029ns) DropOutgoingHandshake:456(473ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:476(2129ns) {SwitchToState:178(740ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.011154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518470309304381:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35876
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.005913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518465741180263:2061] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:37.992698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518466160984513:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.067980Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337076:2079] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.056215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518470354214371:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64531
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:37.992968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518466160984516:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-11-11T17:18:38.005969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518465741180263:2061] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.056330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518470354214372:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64531
2025-11-11T17:18:38.148559Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337072:2076] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.275126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518462264007043:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(210901986ns) {Configure:104(210897104ns) {ProcessConfigured:135(79214ns) {StartInitialHandshake:157(76747ns) {DropHandshakes:469(2306ns) {DropIncomingHandshake:441(449ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(337ns) PrepareNewSessionHandshake:476(1448ns) {SwitchToState:178(704ns)}}}}}
2025-11-11T17:18:38.275593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518466558974399:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:38.011158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518465741180263:2061] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:38.044942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518466160984516:2078] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.056636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518466059247074:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64531
2025-11-11T17:18:38.148616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518466014337072:2076] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.148728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337075:2078] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.181056Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337068:2074] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.181081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518466014337068:2074] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.305744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518466014337037:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(38793133ns) {IncomingHandshake:261(38790274ns) {DropIncomingHandshake:441(1965ns) PrepareNewSessionHandshake:476(4802ns) {SwitchToState:178(3364ns)} GenerateSessionVirtualId:988(678ns) IssueIncomingHandshakeReply:193(7397ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.317683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518468449058028:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:38.294138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518462264007041:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16929483ns) {Configure:104(16923788ns) {ProcessConfigured:135(76445ns) {StartInitialHandshake:157(73294ns) {DropHandshakes:469(3016ns) {DropIncomingHandshake:441(785ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:476(1818ns) {SwitchToState:178(614ns)}}}}}
2025-11-11T17:18:38.065011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518465741180249:2049] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.045282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518466160984475:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.061786Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518470354214371:2073] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.338367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518466014337072:2076] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.045359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518470455951816:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-11-11T17:18:38.062121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518466059247040:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.065057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:38.338670Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518466014337068:2074] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:38.296262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974400:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.065107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518465741180249:2049] [node 2] ICP22 created new session: [5:7571518470036147567:2049]
2025-11-11T17:18:38.065145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518470036147567:2049] [node 2] ICS09 handshake done sender: [5:7571518465741180263:2061] self: [5:7571518465741180262:0] peer: [2:7571518466160984517:0] socket: 132 qp: -1
2025-11-11T17:18:38.065179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518470036147567:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:38.065212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:38.296446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974401:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.296545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974402:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.334063Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518466558974400:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.345817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518466558974402:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.369672Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518466558974401:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.076507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518470354214384:2083] [node 5] ICP01 ready to work
2025-11-11T17:18:38.148487Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518470354214372:2074] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.242203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518470354214384:2083] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(165431229ns) {Configure:104(165425566ns) {ProcessConfigured:135(68337785ns) {StartInitialHandshake:157(68334494ns) {DropHandshakes:469(3392ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(663ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:476(8043ns) {SwitchToState:178(2670ns)}}}}}
2025-11-11T17:18:38.274512Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518466059247074:2072] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.274640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518466059247041:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.338331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518470354214421:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64531
2025-11-11T17:18:38.338381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518470354214422:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64531
2025-11-11T17:18:38.338769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518466059247061:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(24371626ns) {IncomingHandshake:261(24369099ns) {DropIncomingHandshake:441(586ns) PrepareNewSessionHandshake:476(2832ns) {SwitchToState:178(1010ns)} GenerateSessionVirtualId:988(3109ns) IssueIncomingHandshakeReply:193(9276ns)}}
2025-11-11T17:18:38.274044Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518466160984501:2071] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:38.301769Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518466160984508:2075] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.320424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518466160984513:2076] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.320465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:38.320509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518466160984513:2076] [node 5] ICP22 created new session: [2:7571518470455951856:2076]
2025-11-11T17:18:38.320549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518470455951856:2076] [node 5] ICS09 handshake done sender: [2:7571518466160984501:2071] self: [2:7571518466160984517:0] peer: [5:7571518465741180262:0] socket: 133 qp: -1
2025-11-11T17:18:38.320599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518470455951856:2076] [node 5] ICS10 traffic start
2025-11-11T17:18:38.320632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:38.321442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518470455951855:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-11-11T17:18:38.339042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518466160984505:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.339080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518466160984505:2073] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.359690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518470455951860:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-11-11T17:18:38.370083Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518466160984505:2073] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:38.386335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518466160984476:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.386405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:38.386456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518466160984476:2049] [node 3] ICP22 created new session: [2:7571518470455951861:2049]
2025-11-11T17:18:38.386501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518470455951861:2049] [node 3] ICS09 handshake done sender: [2:7571518466160984505:2073] self: [2:7571518466160984504:0] peer: [3:7571518470354214418:0] socket: 139 qp: -1
2025-11-11T17:18:38.386540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518470455951861:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:38.386584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:38.387120Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518466160984516:2078] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.124469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518470036147581:2072] [node 6] ICP01 ready to work
2025-11-11T17:18:38.407378Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518470309304381:2083] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.407425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518466014337038:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.407479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:38.407523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518466014337038:2049] [node 3] ICP22 created new session: [1:7571518470309304422:2049]
2025-11-11T17:18:38.394808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518464154090661:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(97145200ns) {Configure:104(88978789ns) {ProcessConfigured:135(75379ns) {StartInitialHandshake:157(72492ns) {DropHandshakes:469(3909ns) {DropIncomingHandshake:441(720ns) DropOutgoingHandshake:456(1062ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:476(2565ns) {SwitchToState:178(511ns)}}}}}
2025-11-11T17:18:38.407562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518470309304422:2049] [node 3] ICS09 handshake done sender: [1:7571518466014337072:2076] self: [1:7571518466014337071:0] peer: [3:7571518470354214375:0] socket: 145 qp: -1
2025-11-11T17:18:38.124762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518470036147582:2074] [node 4] ICP01 ready to work
2025-11-11T17:18:38.398972Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518470354214371:2073] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:38.407595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518470309304422:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:38.407626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:38.408558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518470309304421:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35876
2025-11-11T17:18:38.408710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518466014337037:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.408783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:38.408817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518466014337037:2048] [node 2] ICP22 created new session: [1:7571518470309304424:2048]
2025-11-11T17:18:38.408860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518470309304424:2048] [node 2] ICS09 handshake done sender: [1:7571518466014337068:2074] self: [1:7571518466014337067:0] peer: [2:7571518470455951817:0] socket: 143 qp: -1
2025-11-11T17:18:38.408890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518470309304424:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:38.408917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:38.441965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518466160984475:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.442031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:38.442073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518466160984475:2048] [node 1] ICP22 created new session: [2:7571518470455951863:2048]
2025-11-11T17:18:38.442116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518470455951863:2048] [node 1] ICS09 handshake done sender: [2:7571518466160984516:2078] self: [2:7571518470455951817:0] peer: [1:7571518466014337067:0] socket: 151 qp: -1
2025-11-11T17:18:38.442154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518470455951863:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:38.442191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:38.442875Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.450020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518466160984475:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:38.450344Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518466160984507:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.450371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518466160984507:2074] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.451734Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518466160984507:2074] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:38.452018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518466160984496:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.452084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:38.452126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518466160984496:2066] [node 6] ICP22 created new session: [2:7571518470455951867:2066]
2025-11-11T17:18:38.452172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518470455951867:2066] [node 6] ICS09 handshake done sender: [2:7571518466160984507:2074] self: [2:7571518466160984506:0] peer: [6:7571518466558974418:0] socket: 140 qp: -1
2025-11-11T17:18:38.452208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518470455951867:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:38.452238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:38.417657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518466558974415:2067] [node 5] ICP01 ready to work
2025-11-11T17:18:38.399126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518470354214423:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64531
2025-11-11T17:18:38.124921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518470036147569:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-11T17:18:38.409916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518464154090680:2064] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.425888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518462264007043:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.399182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518470354214424:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64531
2025-11-11T17:18:38.422004Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.422902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518466059247040:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.422980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:38.423027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518466059247040:2048] [node 1] ICP22 created new session: [3:7571518470354214425:2048]
2025-11-11T17:18:38.423066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518470354214425:2048] [node 1] ICS09 handshake done sender: [3:7571518470354214371:2073] self: [3:7571518470354214375:0] peer: [1:7571518466014337071:0] socket: 150 qp: -1
2025-11-11T17:18:38.423105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518470354214425:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:38.423139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:38.423652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518466059247041:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518466059247074:2072] held: no
2025-11-11T17:18:38.444641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518470354214423:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.444807Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518466059247074:2072] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:38.454328Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518466059247069:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.454349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518466059247069:2070] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.454807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518466059247041:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.454877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:38.454920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518466059247041:2049] [node 2] ICP22 created new session: [3:7571518470354214428:2049]
2025-11-11T17:18:38.454963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518470354214428:2049] [node 2] ICS09 handshake done sender: [3:7571518466059247074:2072] self: [3:7571518470354214418:0] peer: [2:7571518466160984504:0] socket: 149 qp: -1
2025-11-11T17:18:38.454994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518470354214428:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:38.455022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.477289Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518470354214421:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.133728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518465741180248:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(299556384ns) {Configure:104(299551319ns) {ProcessConfigured:135(24591286ns) {StartInitialHandshake:157(24588616ns) {DropHandshakes:469(2359ns) {DropIncomingHandshake:441(605ns) DropOutgoingHandshake:456(369ns)} GenerateSessionVirtualId:988(862ns) PrepareNewSessionHandshake:476(4053ns) {SwitchToState:178(1519ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.409940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518464154090680:2064] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.477528Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518466059247069:2070] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:38.480339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518466059247064:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.480437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:38.480485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518466059247064:2066] [node 6] ICP22 created new session: [3:7571518470354214432:2066]
2025-11-11T17:18:38.480532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518470354214432:2066] [node 6] ICS09 handshake done sender: [3:7571518466059247069:2070] self: [3:7571518466059247068:0] peer: [6:7571518466558974416:0] socket: 136 qp: -1
2025-11-11T17:18:38.480566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518470354214432:2066] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.462191Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337070:2075] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.481970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518470455951855:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.425958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518462264007041:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.494202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518466160984496:2066] [node 6] ICP18 handshake (actor [2:7571518470455951855:2119]) is too old
2025-11-11T17:18:38.494308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518470455951869:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-11-11T17:18:38.426001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518462264007042:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.466797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974419:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.350572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518470036147582:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(201944509ns) {Configure:104(136541351ns) {ProcessConfigured:135(68475ns) {StartInitialHandshake:157(66789ns) {DropHandshakes:469(2337ns) {DropIncomingHandshake:441(615ns) DropOutgoingHandshake:456(426ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:476(1919ns) {SwitchToState:178(691ns)}}}}}
2025-11-11T17:18:38.393779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518470036147581:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(243248811ns) {Configure:104(243243191ns) {ProcessConfigured:135(42462491ns) {StartInitialHandshake:157(42459317ns) {DropHandshakes:469(13443751ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(2782ns)} GenerateSessionVirtualId:988(2615ns) PrepareNewSessionHandshake:476(5761ns) {SwitchToState:178(2136ns)}}}}}
2025-11-11T17:18:38.406740Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518470036147569:2065] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.462226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518466014337070:2075] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.480617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.410845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058032:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.519307Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518470455951869:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.521661Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518466160984515:2077] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.521704Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518466160984515:2077] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.413864Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518464154090680:2064] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:38.419842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518468449058028:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(77092317ns) {Configure:104(77083251ns) {ProcessConfigured:135(74744ns) {StartInitialHandshake:157(72692ns) {DropHandshakes:469(3301ns) {DropIncomingHandshake:441(832ns) DropOutgoingHandshake:456(550ns)} GenerateSessionVirtualId:988(930ns) PrepareNewSessionHandshake:476(2076ns) {SwitchToState:178(618ns)}}}}}
2025-11-11T17:18:38.470096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518464154090676:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.470171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:38.470225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518464154090676:2061] [node 5] ICP22 created new session: [4:7571518468449058036:2061]
2025-11-11T17:18:38.470263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518468449058036:2061] [node 5] ICS09 handshake done sender: [4:7571518464154090680:2064] self: [4:7571518464154090679:0] peer: [5:7571518470036147627:0] socket: 157 qp: -1
2025-11-11T17:18:38.470300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518468449058036:2061] [node 5] ICS10 traffic start
2025-11-11T17:18:38.470335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:38.476238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058039:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.476413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058040:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.476461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058041:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.481644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518468449058040:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.481766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518464154090662:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.489720Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518468449058039:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.499841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518464154090661:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.500043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058043:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.506135Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518468449058032:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.509720Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518468449058041:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.516281Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.516289Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518468449058040:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:38.406865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518470036147582:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.531137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518464154090662:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518468449058040:2117] held: no
2025-11-11T17:18:38.531243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518464154090662:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.531326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:38.531369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518464154090662:2050] [node 3] ICP22 created new session: [4:7571518468449058045:2050]
2025-11-11T17:18:38.531431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518468449058045:2050] [node 3] ICS09 handshake done sender: [4:7571518468449058040:2117] self: [4:7571518468449058042:0] peer: [3:7571518466059247072:0] socket: 185 qp: -1
2025-11-11T17:18:38.531466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518468449058045:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:38.531504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:38.531943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518464154090676:2061] [node 5] ICP18 handshake (actor [4:7571518468449058032:2114]) is too old
2025-11-11T17:18:38.532030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518464154090660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.538573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974420:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.480920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518466059247064:2066] [node 6] ICP18 handshake (actor [3:7571518470354214421:2118]) is too old
2025-11-11T17:18:38.514167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518470036147628:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-11T17:18:38.518862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518470036147629:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-11T17:18:38.519011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518470036147630:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-11T17:18:38.530636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518470036147629:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.530732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518470036147630:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.530824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518470036147569:2065] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:38.532903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518470036147582:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.532949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:38.532994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518470036147582:2074] [node 4] ICP22 created new session: [5:7571518470036147631:2074]
2025-11-11T17:18:38.533040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518470036147631:2074] [node 4] ICS09 handshake done sender: [5:7571518470036147569:2065] self: [5:7571518470036147627:0] peer: [4:7571518464154090679:0] socket: 158 qp: -1
2025-11-11T17:18:38.533072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518470036147631:2074] [node 4] ICS10 traffic start
2025-11-11T17:18:38.533103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:38.533608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518465741180248:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.533657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518465741180250:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.473785Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518466014337070:2075] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.486018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518466014337065:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.546779Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518466160984515:2077] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:38.498452Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518466059247073:2071] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.498476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518466059247073:2071] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.486071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:38.505984Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518466059247073:2071] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:38.514427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518466059247061:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.514518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:38.514567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518466059247061:2065] [node 4] ICP22 created new session: [3:7571518470354214436:2065]
2025-11-11T17:18:38.514615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518470354214436:2065] [node 4] ICS09 handshake done sender: [3:7571518466059247073:2071] self: [3:7571518466059247072:0] peer: [4:7571518468449058042:0] socket: 138 qp: -1
2025-11-11T17:18:38.514657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518470354214436:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:38.514691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:38.597635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518470354214414:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.597686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518470354214414:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.558217Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.486113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518466014337065:2072] [node 6] ICP22 created new session: [1:7571518470309304430:2072]
2025-11-11T17:18:38.573952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518466160984499:2067] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.578007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518466558974415:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(156486128ns) {Configure:104(156480779ns) {ProcessConfigured:135(63291ns) {StartInitialHandshake:157(60287ns) {DropHandshakes:469(3577ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(810ns)} GenerateSessionVirtualId:988(1348ns) PrepareNewSessionHandshake:476(2034ns) {SwitchToState:178(616ns)}}}}}
2025-11-11T17:18:38.549817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058048:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.486154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518470309304430:2072] [node 6] ICS09 handshake done sender: [1:7571518466014337070:2075] self: [1:7571518466014337069:0] peer: [6:7571518466558974417:0] socket: 144 qp: -1
2025-11-11T17:18:38.486186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518470309304430:2072] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.486220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:38.490878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518470309304421:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.497828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518466014337065:2072] [node 6] ICP18 handshake (actor [1:7571518470309304421:2119]) is too old
2025-11-11T17:18:38.561656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337074:2077] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.561706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518466014337074:2077] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.569841Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518466014337081:2080] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.569867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518466014337081:2080] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.582962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518466014337081:2080] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:38.602002Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518466014337074:2077] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:38.602238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518466014337066:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.602297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:38.602338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518466014337066:2073] [node 5] ICP22 created new session: [1:7571518470309304434:2073]
2025-11-11T17:18:38.602382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518470309304434:2073] [node 5] ICS09 handshake done sender: [1:7571518466014337081:2080] self: [1:7571518466014337080:0] peer: [5:7571518470036147633:0] socket: 142 qp: -1
2025-11-11T17:18:38.602416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518470309304434:2073] [node 5] ICS10 traffic start
2025-11-11T17:18:38.602452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:38.604249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518466014337064:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.604294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:38.604331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518466014337064:2071] [node 4] ICP22 created new session: [1:7571518470309304436:2071]
2025-11-11T17:18:38.604374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518470309304436:2071] [node 4] ICS09 handshake done sender: [1:7571518466014337074:2077] self: [1:7571518466014337073:0] peer: [4:7571518468449058047:0] socket: 141 qp: -1
2025-11-11T17:18:38.604404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518470309304436:2071] [node 4] ICS10 traffic start
2025-11-11T17:18:38.604434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:38.605072Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518470354214414:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:38.581986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:38.582044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518466160984499:2067] [node 4] ICP22 created new session: [2:7571518470455951872:2067]
2025-11-11T17:18:38.582089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518470455951872:2067] [node 4] ICS09 handshake done sender: [2:7571518466160984515:2077] self: [2:7571518466160984514:0] peer: [4:7571518468449058044:0] socket: 134 qp: -1
2025-11-11T17:18:38.582138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518470455951872:2067] [node 4] ICS10 traffic start
2025-11-11T17:18:38.582173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.606483Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.578129Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.610726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518465741180248:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518470036147629:2117] held: no
2025-11-11T17:18:38.583195Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.586295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518462264007041:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518466558974402:2110] held: no
2025-11-11T17:18:38.586410Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518466558974401:2109] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:38.586717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518462264007043:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518466558974400:2108] held: no
2025-11-11T17:18:38.586972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518462264007042:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518466558974401:2109] held: no
2025-11-11T17:18:38.587047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518462264007042:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.587122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:38.587166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518462264007042:2049] [node 2] ICP22 created new session: [6:7571518466558974425:2049]
2025-11-11T17:18:38.587225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518466558974425:2049] [node 2] ICS09 handshake done sender: [6:7571518466558974401:2109] self: [6:7571518466558974418:0] peer: [2:7571518466160984506:0] socket: 160 qp: -1
2025-11-11T17:18:38.587258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518466558974425:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:38.587291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:38.587684Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518466558974402:2110] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:38.602081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518462264007041:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.617701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:38.617764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518462264007041:2048] [node 1] ICP22 created new session: [6:7571518466558974427:2048]
2025-11-11T17:18:38.617808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518466558974427:2048] [node 1] ICS09 handshake done sender: [6:7571518466558974402:2110] self: [6:7571518466558974417:0] peer: [1:7571518466014337069:0] socket: 161 qp: -1
2025-11-11T17:18:38.617852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518466558974427:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:38.618195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974421:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.622491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.579234Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518468449058039:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:38.605578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518470354214384:2083] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.637956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518462264007041:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(35770839ns) {HandleHandshakeStatus:329(35767511ns) {DropIncomingHandshake:441(984ns) DropOutgoingHandshake:456(401ns) SwitchToState:178(552ns) ProcessPendingSessionEvents:495(992ns)}}
2025-11-11T17:18:38.638791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974424:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.639121Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518466558974400:2108] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:38.646601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974428:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.646750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518462264007043:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.646849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:38.646917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518462264007043:2050] [node 3] ICP22 created new session: [6:7571518466558974432:2050]
2025-11-11T17:18:38.646958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518466558974432:2050] [node 3] ICS09 handshake done sender: [6:7571518466558974400:2108] self: [6:7571518466558974416:0] peer: [3:7571518466059247068:0] socket: 159 qp: -1
2025-11-11T17:18:38.646986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518466558974432:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:38.647014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:38.653897Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518466558974424:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.609810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:38.609858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518470354214384:2083] [node 5] ICP22 created new session: [3:7571518470354214439:2083]
2025-11-11T17:18:38.609906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518470354214439:2083] [node 5] ICS09 handshake done sender: [3:7571518470354214414:2113] self: [3:7571518470354214407:0] peer: [5:7571518470036147634:0] socket: 188 qp: -1
2025-11-11T17:18:38.609945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518470354214439:2083] [node 5] ICS10 traffic start
2025-11-11T17:18:38.609979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:38.663885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518466558974415:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:38.610991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518470036147636:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-11T17:18:38.626423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518470036147637:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-11T17:18:38.630101Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.649917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518465741180250:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518470036147630:2118] held: no
2025-11-11T17:18:38.650293Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518470036147629:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.682016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974435:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.609696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518464154090661:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.670117Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518470036147630:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:38.670597Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518470036147626:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.670621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518470036147626:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.671361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518465741180248:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.671413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:38.671459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518465741180248:2048] [node 1] ICP22 created new session: [5:7571518470036147639:2048]
2025-11-11T17:18:38.671499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518470036147639:2048] [node 1] ICS09 handshake done sender: [5:7571518470036147629:2117] self: [5:7571518470036147633:0] peer: [1:7571518466014337080:0] socket: 184 qp: -1
2025-11-11T17:18:38.671534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518470036147639:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:38.671568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:38.672101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518470036147638:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-11T17:18:38.672756Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518470036147626:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:38.672981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518465741180250:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.673044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:38.673085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518465741180250:2050] [node 3] ICP22 created new session: [5:7571518470036147642:2050]
2025-11-11T17:18:38.673122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518470036147642:2050] [node 3] ICS09 handshake done sender: [5:7571518470036147630:2118] self: [5:7571518470036147634:0] peer: [3:7571518470354214407:0] socket: 189 qp: -1
2025-11-11T17:18:38.673146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518470036147642:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:38.673189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:38.673473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518470036147581:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.677870Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518470036147638:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.685789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:38.685885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518470036147581:2072] [node 6] ICP22 created new session: [5:7571518470036147644:2072]
2025-11-11T17:18:38.685935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518470036147644:2072] [node 6] ICS09 handshake done sender: [5:7571518470036147626:2115] self: [5:7571518470036147619:0] peer: [6:7571518466558974434:0] socket: 172 qp: -1
2025-11-11T17:18:38.685973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518470036147644:2072] [node 6] ICS10 traffic start
2025-11-11T17:18:38.686012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:38.686436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518470036147581:2072] [node 6] ICP18 handshake (actor [5:7571518470036147638:2121]) is too old
2025-11-11T17:18:38.645717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:38.645774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518464154090661:2049] [node 2] ICP22 created new session: [4:7571518468449058050:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.645825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518468449058050:2049] [node 2] ICS09 handshake done sender: [4:7571518468449058039:2116] self: [4:7571518468449058044:0] peer: [2:7571518466160984514:0] socket: 182 qp: -1
2025-11-11T17:18:38.645867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518468449058050:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:38.645908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:38.650378Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.653926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518464154090661:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(40340974ns) {HandleHandshakeStatus:329(40338518ns) {DropIncomingHandshake:441(1788ns) DropOutgoingHandshake:456(6357ns) SwitchToState:178(3820ns) ProcessPendingSessionEvents:495(105827ns) {ForwardSessionEventToSession:612(59641ns) ForwardSessionEventToSession:612(3202ns) ForwardSessionEventToSession:612(5825ns) ForwardSessionEventToSession:612(2677ns) ForwardSessionEventToSession:612(8137ns) ForwardSessionEventToSession:612(4440ns) ForwardSessionEventToSession:612(2901ns) ForwardSessionEventToSession:612(2008ns) ForwardSessionEventToSession:612(5431ns) ForwardSessionEventToSession:612(2297ns)}}}
2025-11-11T17:18:38.654170Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.674076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518464154090661:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:38.674555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518464154090660:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518468449058041:2118] held: no
2025-11-11T17:18:38.675884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058052:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.706951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518468449058053:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-11T17:18:38.707324Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518468449058041:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:38.717745Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.718004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518464154090660:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.727926Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518466558974424:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:38.726717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:38.726805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518464154090660:2048] [node 1] ICP22 created new session: [4:7571518468449058054:2048]
2025-11-11T17:18:38.726850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518468449058054:2048] [node 1] ICS09 handshake done sender: [4:7571518468449058041:2118] self: [4:7571518468449058047:0] peer: [1:7571518466014337073:0] socket: 186 qp: -1
2025-11-11T17:18:38.726893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518468449058054:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:38.726941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:38.729701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518466558974428:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.729782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518466558974415:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518466558974424:2121] held: no
2025-11-11T17:18:38.729872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518466558974415:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.729955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:38.730003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518466558974415:2067] [node 5] ICP22 created new session: [6:7571518466558974436:2067]
2025-11-11T17:18:38.730048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518466558974436:2067] [node 5] ICS09 handshake done sender: [6:7571518466558974424:2121] self: [6:7571518466558974434:0] peer: [5:7571518470036147619:0] socket: 164 qp: -1
2025-11-11T17:18:38.730097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518466558974436:2067] [node 5] ICS10 traffic start
2025-11-11T17:18:38.730150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:38.731197Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518468449058053:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.731319Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518468449058035:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:38.731339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518468449058035:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:38.732468Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518468449058035:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:38.732714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518468449058028:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:38.732783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:38.732827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518468449058028:2069] [node 6] ICP22 created new session: [4:7571518468449058058:2069]
2025-11-11T17:18:38.732869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518468449058058:2069] [node 6] ICS09 handshake done sender: [4:7571518468449058035:2115] self: [4:7571518468449058034:0] peer: [6:7571518466558974438:0] socket: 147 qp: -1
2025-11-11T17:18:38.732900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518468449058058:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:38.732935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:38.730810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518466558974399:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:38.746868Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25701) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:38.748491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518466558974399:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25701) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518466558974428:2122] held: no
2025-11-11T17:18:38.748756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518466558974439:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-11T17:18:38.749206Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518466558974428:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:38.749383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518466558974399:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:38.749465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:38.749509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518466558974399:2070] [node 4] ICP22 created new session: [6:7571518466558974440:2070]
2025-11-11T17:18:38.749555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518466558974440:2070] [node 4] ICS09 handshake done sender: [6:7571518466558974428:2122] self: [6:7571518466558974438:0] peer: [4:7571518468449058034:0] socket: 148 qp: -1
2025-11-11T17:18:38.749927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518466558974440:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:38.749977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:38.785790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:38.786035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518470455951863:2048] [node 1] ICS01 socket: 151 reason# EndOfStream
2025-11-11T17:18:38.786069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518466160984475:2048] [node 1] ICP30 unregister session Session# [2:7571518470455951863:2048] VirtualId# [2:7571518470455951817:0]
2025-11-11T17:18:38.786279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518470455951863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.786420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:38.786471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:38.789757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:38.789984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518470354214425:2048] [node 1] ICS01 socket: 150 reason# EndOfStream
2025-11-11T17:18:38.790022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518466059247040:2048] [node 1] ICP30 unregister session Session# [3:7571518470354214425:2048] VirtualId# [3:7571518470354214375:0]
2025-11-11T17:18:38.790054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518470354214425:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.790153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:38.790198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:38.805714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:38.805977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518470036147639:2048] [node 1] ICS01 socket: 184 reason# EndOfStream
2025-11-11T17:18:38.805714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:38.806011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518465741180248:2048] [node 1] ICP30 unregister session Session# [5:7571518470036147639:2048] VirtualId# [5:7571518470036147633:0]
2025-11-11T17:18:38.805977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518468449058054:2048] [node 1] ICS01 socket: 186 reason# EndOfStream
2025-11-11T17:18:38.806041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518470036147639:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.806013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518464154090660:2048] [node 1] ICP30 unregister session Session# [4:7571518468449058054:2048] VirtualId# [4:7571518468449058047:0]
2025-11-11T17:18:38.806120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:38.806042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518468449058054:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.806172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:38.806125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:38.806173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:38.806497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:38.806725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518466558974427:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-11-11T17:18:38.806755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518462264007041:2048] [node 1] ICP30 unregister session Session# [6:7571518466558974427:2048] VirtualId# [6:7571518466558974417:0]
2025-11-11T17:18:38.806777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518466558974427:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.806837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:38.806868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:38.855233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:38.855451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518470354214428:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-11T17:18:38.855490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518466059247041:2049] [node 2] ICP30 unregister session Session# [3:7571518470354214428:2049] VirtualId# [3:7571518470354214418:0]
2025-11-11T17:18:38.855531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518470354214428:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.855610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:38.855653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:38.867149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:38.867391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518466558974425:2049] [node 2] ICS01 socket: 160 reason# EndOfStream
2025-11-11T17:18:38.867432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518462264007042:2049] [node 2] ICP30 unregister session Session# [6:7571518466558974425:2049] VirtualId# [6:7571518466558974418:0]
2025-11-11T17:18:38.867463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518466558974425:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.867551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:38.867597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:38.873774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:38.874020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518470036147567:2049] [node 2] ICS07 socket disconnect 132 reason# EndOfStream
2025-11-11T17:18:38.874044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518470036147567:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.874130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:38.874179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518470036147567:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:38.873832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:38.874035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518468449058050:2049] [node 2] ICS07 socket disconnect 182 reason# EndOfStream
2025-11-11T17:18:38.874067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518468449058050:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.874145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:38.874191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518468449058050:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:38.875458Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address
2025-11-11T17:18:38.878092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518465741180249:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:38.878396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518470036147567:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:38.878442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518465741180249:2049] [node 2] ICP30 unregister session Session# [5:7571518470036147567:2049] VirtualId# [5:7571518465741180262:0]
2025-11-11T17:18:38.878492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:38.878530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518465741180249:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:38.878544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:38.878732Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address
2025-11-11T17:18:38.879060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518464154090661:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:38.879108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518468449058050:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:38.879137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518464154090661:2049] [node 2] ICP30 unregister session Session# [4:7571518468449058050:2049] VirtualId# [4:7571518468449058044:0]
2025-11-11T17:18:38.879183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:38.879207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518464154090661:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:38.879223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:38.888341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518465741180249:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:38.893767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518464154090661:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:38.985885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:38.986067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518470036147642:2050] [node 3] ICS01 socket: 189 reason# EndOfStream
2025-11-11T17:18:38.986105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518465741180250:2050] [node 3] ICP30 unregister session Session# [5:7571518470036147642:2050] VirtualId# [5:7571518470036147634:0]
2025-11-11T17:18:38.986127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518470036147642:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.986202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:38.986237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:38.985062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:38.985333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518466558974432:2050] [node 3] ICS01 socket: 159 reason# EndOfStream
2025-11-11T17:18:38.985369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518462264007043:2050] [node 3] ICP30 unregister session Session# [6:7571518466558974432:2050] VirtualId# [6:7571518466558974416:0]
2025-11-11T17:18:38.985403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518466558974432:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.985513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:38.985557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:38.993492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:38.993770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518468449058045:2050] [node 3] ICS01 socket: 185 reason# EndOfStream
2025-11-11T17:18:38.993811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518464154090662:2050] [node 3] ICP30 unregister session Session# [4:7571518468449058045:2050] VirtualId# [4:7571518468449058042:0]
2025-11-11T17:18:38.993844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518468449058045:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:38.993977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:38.994024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:39.085834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:39.086201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518470036147631:2074] [node 4] ICS01 socket: 158 reason# EndOfStream
2025-11-11T17:18:39.086296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518470036147582:2074] [node 4] ICP30 unregister session Session# [5:7571518470036147631:2074] VirtualId# [5:7571518470036147627:0]
2025-11-11T17:18:39.086369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518470036147631:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:39.086572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:39.086700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:39.086899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:39.087321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518466558974440:2070] [node 4] ICS01 socket: 148 reason# EndOfStream
2025-11-11T17:18:39.087387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518466558974399:2070] [node 4] ICP30 unregister session Session# [6:7571518466558974440:2070] VirtualId# [6:7571518466558974438:0]
2025-11-11T17:18:39.087432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518466558974440:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:39.087548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:39.087630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:39.143828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:39.144080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518466558974436:2067] [node 5] ICS01 socket: 164 reason# EndOfStream
2025-11-11T17:18:39.144122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518466558974415:2067] [node 5] ICP30 unregister session Session# [6:7571518466558974436:2067] VirtualId# [6:7571518466558974434:0]
2025-11-11T17:18:39.144182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518466558974436:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:39.144310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:39.144356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:39.306081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518472841047531:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:39.306132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518472841047532:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:39.365952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518471746070908:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:39.365999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518471746070909:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:39.401889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518472019139640:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:39.401932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518472019139641:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:39.465880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518475074500410:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:39.465927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518475074500411:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:39.465950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518475074500412:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:39.537657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518472304674550:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:39.537708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518472304674551:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:39.537733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518472304674552:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:39.979713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518472019139663:2069] [node 5] ICP01 ready to work
2025-11-11T17:18:39.980018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518472019139664:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:40.129798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518479369467738:2068] [node 6] ICP01 ready to work
2025-11-11T17:18:40.226076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518479369467767:2076] [node 5] ICP01 ready to work
2025-11-11T17:18:39.725890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518470997837771:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:40.197859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518477136014890:2107] [node 5] ICP01 ready to work
2025-11-11T17:18:40.295347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518472304674551:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(265228899ns) {Configure:104(265224311ns) {ProcessConfigured:135(59573ns) {StartInitialHandshake:157(56770ns) {DropHandshakes:469(2643ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(367ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:476(1123ns) {SwitchToState:178(448ns)}}}}}
2025-11-11T17:18:39.725938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518470997837772:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:40.239090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518477136014891:2109] [node 6] ICP01 ready to work
2025-11-11T17:18:40.274079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518476041038268:2075] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.298213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518472019139663:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(317960562ns) {Configure:104(317956392ns) {ProcessConfigured:135(56217984ns) {StartInitialHandshake:157(20280473ns) {DropHandshakes:469(3938ns) {DropIncomingHandshake:441(767ns) DropOutgoingHandshake:456(655ns)} GenerateSessionVirtualId:988(1846ns) PrepareNewSessionHandshake:476(3875ns) {SwitchToState:178(1738ns)}}}}}
2025-11-11T17:18:40.298608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518476314107003:2071] [node 6] ICP01 ready to work
2025-11-11T17:18:39.725963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518470997837773:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:40.277734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518476041038267:2076] [node 4] ICP01 ready to work
2025-11-11T17:18:40.295764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518476599641911:2069] [node 6] ICP01 ready to work
2025-11-11T17:18:40.322170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518475292805128:2061] [node 5] ICP01 ready to work
2025-11-11T17:18:40.330212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518475292805132:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.238920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518477136014892:2108] [node 4] ICP01 ready to work
2025-11-11T17:18:40.301822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518476599641915:2066] [node 4] ICP01 ready to work
2025-11-11T17:18:40.372831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805133:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.349819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518476041038270:2077] [node 5] ICP01 ready to work
2025-11-11T17:18:40.378129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518475074500412:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(336662208ns) {Configure:104(336657321ns) {ProcessConfigured:135(57921ns) {StartInitialHandshake:157(55313ns) {DropHandshakes:469(2180ns) {DropIncomingHandshake:441(475ns) DropOutgoingHandshake:456(367ns)} GenerateSessionVirtualId:988(655ns) PrepareNewSessionHandshake:476(1401ns) {SwitchToState:178(544ns)}}}}}
2025-11-11T17:18:40.358165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518472019139640:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(40035169ns) {Configure:104(40029252ns) {ProcessConfigured:135(67811ns) {StartInitialHandshake:157(63270ns) {DropHandshakes:469(642ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(187ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(2177ns) {SwitchToState:178(595ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.407150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467780:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.401938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518472019139664:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(404074790ns) {Configure:104(404058960ns) {ProcessConfigured:135(23395083ns) {StartInitialHandshake:157(23391248ns) {DropHandshakes:469(1205ns) {DropIncomingHandshake:441(295ns) DropOutgoingHandshake:456(273ns)} GenerateSessionVirtualId:988(397ns) PrepareNewSessionHandshake:476(4542ns) {SwitchToState:178(2372ns)}}}}}
2025-11-11T17:18:40.372921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805135:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.439330Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518479369467780:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.326939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518476599641911:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(29542585ns) {Configure:104(29538529ns) {ProcessConfigured:135(75352ns) {StartInitialHandshake:157(73214ns) {DropHandshakes:469(3756ns) {DropIncomingHandshake:441(784ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(1954ns) PrepareNewSessionHandshake:476(1987ns) {SwitchToState:178(657ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.457760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518471746070909:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(383915927ns) {Configure:104(383911303ns) {ProcessConfigured:135(37856268ns) {StartInitialHandshake:157(37854426ns) {DropHandshakes:469(656ns) {DropIncomingHandshake:441(104ns) DropOutgoingHandshake:456(205ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:476(6369ns) {SwitchToState:178(2444ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.459337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518476041038268:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(41372039ns) {Configure:104(41366106ns) {ProcessConfigured:135(13439242ns) {StartInitialHandshake:157(13435629ns) {DropHandshakes:469(4353891ns) {DropIncomingHandshake:441(581ns) DropOutgoingHandshake:456(2327ns)} GenerateSessionVirtualId:988(3029ns) PrepareNewSessionHandshake:476(2875ns) {SwitchToState:178(966ns)}}}}}
2025-11-11T17:18:40.461300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518476041038274:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60210
2025-11-11T17:18:40.461420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518476041038275:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60210
2025-11-11T17:18:40.461506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518476041038276:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.373088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805136:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.429901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518472019139641:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(417095897ns) {Configure:104(417066475ns) {ProcessConfigured:135(40108437ns) {StartInitialHandshake:157(40105054ns) {DropHandshakes:469(3714ns) {DropIncomingHandshake:441(889ns) DropOutgoingHandshake:456(705ns)} GenerateSessionVirtualId:988(1006ns) PrepareNewSessionHandshake:476(3745ns) {SwitchToState:178(1440ns)}}}}}
2025-11-11T17:18:40.328582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641914:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.439513Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518479369467774:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.446575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518477136014899:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53365
2025-11-11T17:18:40.490727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518471746070908:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(191463296ns) {Configure:104(191457691ns) {ProcessConfigured:135(54459ns) {StartInitialHandshake:157(51953ns) {DropHandshakes:469(2170ns) {DropIncomingHandshake:441(499ns) DropOutgoingHandshake:456(461ns)} GenerateSessionVirtualId:988(561ns) PrepareNewSessionHandshake:476(1583ns) {SwitchToState:178(754ns)}}}}}
2025-11-11T17:18:40.490922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038274:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.463620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518476314107015:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35435
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.398904Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518475292805135:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.446720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518477136014900:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53365
2025-11-11T17:18:40.439536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518479369467774:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.438206Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518476599641914:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.463669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518476314107016:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35435
2025-11-11T17:18:40.525890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038275:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.560065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518476041038285:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60210
2025-11-11T17:18:40.399057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518475292805128:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.440603Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518479369467774:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:40.438584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641923:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.438633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641924:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.438774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518476599641915:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.450251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518476599641917:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.450301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518476599641917:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.463020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518476599641924:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.498697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518476599641917:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:40.499326Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64916) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:40.518771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518476599641915:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64916) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518476599641914:2112] held: no
2025-11-11T17:18:40.519031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.540045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641931:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.540208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641932:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.540273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518476599641911:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.540325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:40.540374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518476599641911:2069] [node 6] ICP22 created new session: [5:7571518476599641934:2069]
2025-11-11T17:18:40.540422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518476599641934:2069] [node 6] ICS09 handshake done sender: [5:7571518476599641917:2113] self: [5:7571518476599641916:0] peer: [6:7571518475292805144:0] socket: 143 qp: -1
2025-11-11T17:18:40.540450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518476599641934:2069] [node 6] ICS10 traffic start
2025-11-11T17:18:40.540483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:40.553159Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518476599641914:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:40.555042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518476599641931:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.555171Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518476599641923:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.556698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518472304674550:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.473990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014899:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.486150Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107015:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.440911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518479369467767:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.571030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518476599641915:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.571135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:40.571187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518476599641915:2066] [node 4] ICP22 created new session: [5:7571518476599641938:2066]
2025-11-11T17:18:40.571231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518476599641938:2066] [node 4] ICS09 handshake done sender: [5:7571518476599641914:2112] self: [5:7571518476599641925:0] peer: [4:7571518479369467773:0] socket: 141 qp: -1
2025-11-11T17:18:40.571269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518476599641938:2066] [node 4] ICS10 traffic start
2025-11-11T17:18:40.571328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:40.583341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641940:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.406445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518470997837771:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(336385977ns) {Configure:104(336380760ns) {ProcessConfigured:135(84441ns) {StartInitialHandshake:157(81914ns) {DropHandshakes:469(2702ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(481ns)} GenerateSessionVirtualId:988(1227ns) PrepareNewSessionHandshake:476(2157ns) {SwitchToState:178(631ns)}}}}}
2025-11-11T17:18:40.494551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518477136014891:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(132531079ns) {Configure:104(132522799ns) {ProcessConfigured:135(75040ns) {StartInitialHandshake:157(72131ns) {DropHandshakes:469(8068ns) {DropIncomingHandshake:441(665ns) DropOutgoingHandshake:456(644ns)} GenerateSessionVirtualId:988(738ns) PrepareNewSessionHandshake:476(2282ns) {SwitchToState:178(692ns)}}}}}
2025-11-11T17:18:40.560457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518476314107018:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35435
2025-11-11T17:18:40.586446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518476599641932:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.441009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:40.582415Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038276:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.419150Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518475292805133:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.589694Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107018:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.582829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518471746070908:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.585886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038285:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.625788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038271:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.625830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518476041038271:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.630432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518476041038267:2076] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(140590504ns) {Configure:104(140579188ns) {ProcessConfigured:135(66452ns) {StartInitialHandshake:157(64682ns) {DropHandshakes:469(2391ns) {DropIncomingHandshake:441(548ns) DropOutgoingHandshake:456(366ns)} GenerateSessionVirtualId:988(695ns) PrepareNewSessionHandshake:476(1532ns) {SwitchToState:178(437ns)}}}}}
2025-11-11T17:18:40.638533Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038273:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.638556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518476041038273:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.640789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518476041038273:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:40.522994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518477136014907:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53365
2025-11-11T17:18:40.602678Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518476599641931:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:40.441059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518479369467767:2076] [node 5] ICP22 created new session: [4:7571518479369467786:2076]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.470633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805148:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.470696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805149:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.474064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518475292805132:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.506175Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518475292805135:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:40.506389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518475292805128:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.506474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:40.506527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518475292805128:2061] [node 5] ICP22 created new session: [6:7571518475292805152:2061]
2025-11-11T17:18:40.506581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518475292805152:2061] [node 5] ICS09 handshake done sender: [6:7571518475292805135:2112] self: [6:7571518475292805144:0] peer: [5:7571518476599641916:0] socket: 147 qp: -1
2025-11-11T17:18:40.506619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518475292805152:2061] [node 5] ICS10 traffic start
2025-11-11T17:18:40.506675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:40.551493Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518475292805136:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.649824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518476041038268:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.589734Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107016:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.441118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518479369467786:2076] [node 5] ICS09 handshake done sender: [4:7571518479369467774:2110] self: [4:7571518479369467773:0] peer: [5:7571518476599641925:0] socket: 140 qp: -1
2025-11-11T17:18:40.602861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518472304674550:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.551686Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64916) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:40.539603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518472841047532:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(265471092ns) {Configure:104(265464959ns) {ProcessConfigured:135(56317ns) {StartInitialHandshake:157(53948ns) {DropHandshakes:469(2242ns) {DropIncomingHandshake:441(650ns) DropOutgoingHandshake:456(422ns)} GenerateSessionVirtualId:988(1037ns) PrepareNewSessionHandshake:476(1364ns) {SwitchToState:178(492ns)}}}}}
2025-11-11T17:18:40.441156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518479369467786:2076] [node 5] ICS10 traffic start
2025-11-11T17:18:40.649951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:40.602908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:40.609638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107008:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.553672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518470997837773:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(445137819ns) {Configure:104(445128602ns) {ProcessConfigured:135(32688814ns) {StartInitialHandshake:157(32686181ns) {DropHandshakes:469(1397ns) {DropIncomingHandshake:441(351ns) DropOutgoingHandshake:456(312ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:476(4018ns) {SwitchToState:178(1613ns)}}}}}
2025-11-11T17:18:40.557446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518475292805132:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64916) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518475292805133:2111] held: no
2025-11-11T17:18:40.557778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518470997837773:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.557865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805155:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.542903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518477136014910:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53365
2025-11-11T17:18:40.590547Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518475292805133:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:40.593950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518475292805132:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.594001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:40.594058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518475292805132:2066] [node 4] ICP22 created new session: [6:7571518475292805158:2066]
2025-11-11T17:18:40.594108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518475292805158:2066] [node 4] ICS09 handshake done sender: [6:7571518475292805133:2111] self: [6:7571518475292805150:0] peer: [4:7571518479369467768:0] socket: 145 qp: -1
2025-11-11T17:18:40.594139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518475292805158:2066] [node 4] ICS10 traffic start
2025-11-11T17:18:40.594170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:40.637992Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518475292805148:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.638147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518470997837772:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.640568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805162:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.641254Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518475292805148:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:40.641425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518470997837772:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.641476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:40.641515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518470997837772:2049] [node 2] ICP22 created new session: [6:7571518475292805163:2049]
2025-11-11T17:18:40.641556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518475292805163:2049] [node 2] ICS09 handshake done sender: [6:7571518475292805148:2117] self: [6:7571518475292805161:0] peer: [2:7571518476041038272:0] socket: 146 qp: -1
2025-11-11T17:18:40.641599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518475292805163:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:40.641635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:40.692227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805160:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.721677Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27530) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:40.738181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518470997837773:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27530) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518475292805136:2113] held: no
2025-11-11T17:18:40.738367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518475292805136:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:40.753900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805165:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.754058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518470997837773:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.754106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:40.754147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518470997837773:2050] [node 3] ICP22 created new session: [6:7571518475292805166:2050]
2025-11-11T17:18:40.754194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518475292805166:2050] [node 3] ICS09 handshake done sender: [6:7571518475292805136:2113] self: [6:7571518475292805156:0] peer: [3:7571518476314107007:0] socket: 153 qp: -1
2025-11-11T17:18:40.754227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518475292805166:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:40.754271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:40.553703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518477136014892:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(17612890ns) {IncomingHandshake:261(17609448ns) {DropIncomingHandshake:441(412ns) PrepareNewSessionHandshake:476(2201ns) {SwitchToState:178(743ns)} GenerateSessionVirtualId:988(436ns) IssueIncomingHandshakeReply:193(9787ns)}}
2025-11-11T17:18:40.560671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014898:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.560696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518477136014898:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.566206Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518477136014898:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:40.566466Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014900:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.625933Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014910:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.651038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014894:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.651073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518477136014894:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.657930Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014907:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.658002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518477136014890:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.658049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:40.658096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518477136014890:2107] [node 5] ICP22 created new session: [1:7571518477136014913:2107]
2025-11-11T17:18:40.658142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518477136014913:2107] [node 5] ICS09 handshake done sender: [1:7571518477136014898:2112] self: [1:7571518477136014897:0] peer: [5:7571518476599641937:0] socket: 163 qp: -1
2025-11-11T17:18:40.658173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518477136014913:2107] [node 5] ICS10 traffic start
2025-11-11T17:18:40.658210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:40.658938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518477136014890:2107] [node 5] ICP18 handshake (actor [1:7571518477136014910:2118]) is too old
2025-11-11T17:18:40.661808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014896:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.661856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518477136014896:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.691823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518477136014918:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53365
2025-11-11T17:18:40.692871Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518477136014894:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:40.739011Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518477136014896:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:40.762719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518472841047531:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.762905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518477136014892:2108] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.441207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.609691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518476314107008:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.649998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518476041038268:2075] [node 6] ICP22 created new session: [2:7571518476041038292:2075]
2025-11-11T17:18:40.602952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518472304674550:2048] [node 1] ICP22 created new session: [5:7571518476599641941:2048]
2025-11-11T17:18:40.457913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467783:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.602989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518476599641941:2048] [node 1] ICS09 handshake done sender: [5:7571518476599641931:2119] self: [5:7571518476599641937:0] peer: [1:7571518477136014897:0] socket: 177 qp: -1
2025-11-11T17:18:40.610792Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518476314107008:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:40.794662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:40.458972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467790:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.479243Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518479369467783:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.516778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518475074500411:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(384077484ns) {Configure:104(384073164ns) {ProcessConfigured:135(69566ns) {StartInitialHandshake:157(66841ns) {DropHandshakes:469(1942ns) {DropIncomingHandshake:441(483ns) DropOutgoingHandshake:456(373ns)} GenerateSessionVirtualId:988(568ns) PrepareNewSessionHandshake:476(1224ns) {SwitchToState:178(375ns)}}}}}
2025-11-11T17:18:40.517667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518479369467769:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.517717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518479369467769:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.543093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518479369467769:2107] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:40.543358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518479369467738:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.543412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:40.543458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518479369467738:2068] [node 6] ICP22 created new session: [4:7571518479369467794:2068]
2025-11-11T17:18:40.543504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518479369467794:2068] [node 6] ICS09 handshake done sender: [4:7571518479369467769:2107] self: [4:7571518479369467768:0] peer: [6:7571518475292805150:0] socket: 138 qp: -1
2025-11-11T17:18:40.543534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518479369467794:2068] [node 6] ICS10 traffic start
2025-11-11T17:18:40.543567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:40.569682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518479369467790:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.577864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518475074500410:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.669964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467798:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.706052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467799:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.725698Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518479369467798:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.725909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518475074500412:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.778465Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518479369467790:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:40.778604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467801:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.778881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518475074500410:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.778969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:40.779012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518475074500410:2048] [node 1] ICP22 created new session: [4:7571518479369467802:2048]
2025-11-11T17:18:40.779056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518479369467802:2048] [node 1] ICS09 handshake done sender: [4:7571518479369467790:2117] self: [4:7571518479369467797:0] peer: [1:7571518477136014895:0] socket: 165 qp: -1
2025-11-11T17:18:40.779090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518479369467802:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:40.779124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:40.650045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518476041038292:2075] [node 6] ICS09 handshake done sender: [2:7571518476041038273:2111] self: [2:7571518476041038272:0] peer: [6:7571518475292805161:0] socket: 171 qp: -1
2025-11-11T17:18:40.812560Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518475292805165:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.812704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518470997837771:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.815651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518470997837772:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(104546972ns) {HandleHandshakeStatus:329(104544882ns) {DropIncomingHandshake:441(1072ns) DropOutgoingHandshake:456(6259ns) SwitchToState:178(420ns) ProcessPendingSessionEvents:495(83894647ns) {ForwardSessionEventToSession:612(51999918ns) ForwardSessionEventToSession:612(6467ns) ForwardSessionEventToSession:612(7773829ns) ForwardSessionEventToSession:612(8435281ns) ForwardSessionEventToSession:612(11263ns) ForwardSessionEventToSession:612(8459650ns) ForwardSessionEventToSession:612(4504ns) ForwardSessionEventToSession:612(3518ns) ForwardSessionEventToSession:612(7172876ns) ForwardSessionEventToSession:612(2889ns)}}}
2025-11-11T17:18:40.611093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518476314107003:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.603021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518476599641941:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:40.814961Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518479369467801:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.815114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518475074500411:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.802112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518472841047531:2048] [node 2] ICP22 created new session: [1:7571518477136014921:2048]
2025-11-11T17:18:40.603049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:40.614213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518472304674551:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.617656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518472304674552:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(426216229ns) {Configure:104(426214271ns) {ProcessConfigured:135(29279907ns) {StartInitialHandshake:157(29278495ns) {DropHandshakes:469(2462ns) {DropIncomingHandshake:441(561ns) DropOutgoingHandshake:456(364ns)} GenerateSessionVirtualId:988(1017ns) PrepareNewSessionHandshake:476(3605ns) {SwitchToState:178(1745ns)}}}}}
2025-11-11T17:18:40.645828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518472304674552:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.709443Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27530) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:40.710939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518472304674552:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27530) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518476599641923:2115] held: no
2025-11-11T17:18:40.711137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641945:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.711220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518476599641946:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48644
2025-11-11T17:18:40.711520Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518476599641923:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:40.711654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518472304674552:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.711707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:40.711744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518472304674552:2050] [node 3] ICP22 created new session: [5:7571518476599641947:2050]
2025-11-11T17:18:40.711802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518476599641947:2050] [node 3] ICS09 handshake done sender: [5:7571518476599641923:2115] self: [5:7571518476599641944:0] peer: [3:7571518476314107001:0] socket: 151 qp: -1
2025-11-11T17:18:40.711833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518476599641947:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:40.711872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:40.741691Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518476599641932:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:40.742430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518472304674551:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.742482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:40.742527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518472304674551:2049] [node 2] ICP22 created new session: [5:7571518476599641949:2049]
2025-11-11T17:18:40.742574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518476599641949:2049] [node 2] ICS09 handshake done sender: [5:7571518476599641932:2120] self: [5:7571518476599641943:0] peer: [2:7571518476041038277:0] socket: 178 qp: -1
2025-11-11T17:18:40.742609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518476599641949:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:40.742641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.650083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518476041038292:2075] [node 6] ICS10 traffic start
2025-11-11T17:18:40.611134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:40.806923Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 2] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:40.807596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518477136014921:2048] [node 2] ICS09 handshake done sender: [1:7571518477136014894:2110] self: [1:7571518477136014893:0] peer: [2:7571518476041038287:0] socket: 161 qp: -1
2025-11-11T17:18:40.810087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518477136014921:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:40.810180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:40.810622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518472841047531:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(64164454ns) {HandleHandshakeStatus:329(64162182ns) {DropIncomingHandshake:441(1464ns) DropOutgoingHandshake:456(1038ns) SwitchToState:178(533ns) ProcessPendingSessionEvents:495(92531ns) {ForwardSessionEventToSession:612(45817ns) ForwardSessionEventToSession:612(8635ns) ForwardSessionEventToSession:612(3217ns) ForwardSessionEventToSession:612(2419ns) ForwardSessionEventToSession:612(6854ns) ForwardSessionEventToSession:612(2403ns) ForwardSessionEventToSession:612(2439ns) ForwardSessionEventToSession:612(4364ns) ForwardSessionEventToSession:612(1502ns) ForwardSessionEventToSession:612(4964ns)}}}
2025-11-11T17:18:40.810730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518472841047531:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:40.812971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:40.813467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518477136014892:2108] [node 4] ICP22 created new session: [1:7571518477136014922:2108]
2025-11-11T17:18:40.815759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518477136014922:2108] [node 4] ICS09 handshake done sender: [1:7571518477136014896:2111] self: [1:7571518477136014895:0] peer: [4:7571518479369467797:0] socket: 162 qp: -1
2025-11-11T17:18:40.815815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518477136014922:2108] [node 4] ICS10 traffic start
2025-11-11T17:18:40.815859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:40.816241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518477136014892:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(53150929ns) {HandleHandshakeStatus:329(53148897ns) {DropIncomingHandshake:441(14463ns) DropOutgoingHandshake:456(128ns) SwitchToState:178(124ns) ProcessPendingSessionEvents:495(93862ns) {ForwardSessionEventToSession:612(40957ns) ForwardSessionEventToSession:612(6804ns) ForwardSessionEventToSession:612(2860ns) ForwardSessionEventToSession:612(3173ns) ForwardSessionEventToSession:612(5527ns) ForwardSessionEventToSession:612(7519ns) ForwardSessionEventToSession:612(4804ns) ForwardSessionEventToSession:612(2690ns) ForwardSessionEventToSession:612(2299ns) ForwardSessionEventToSession:612(7268ns)}}}
2025-11-11T17:18:40.816334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014905:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.816348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518477136014905:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.838091Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518477136014905:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:40.611174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518476314107003:2071] [node 6] ICP22 created new session: [3:7571518476314107022:2071]
2025-11-11T17:18:40.611213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518476314107022:2071] [node 6] ICS09 handshake done sender: [3:7571518476314107008:2112] self: [3:7571518476314107007:0] peer: [6:7571518475292805156:0] socket: 150 qp: -1
2025-11-11T17:18:40.611247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518476314107022:2071] [node 6] ICS10 traffic start
2025-11-11T17:18:40.611299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:40.613964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518472019139641:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.625751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518472019139641:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24122852ns) {IncomingHandshake:261(24120694ns) {DropIncomingHandshake:441(650ns) PrepareNewSessionHandshake:476(2795ns) {SwitchToState:178(980ns)} GenerateSessionVirtualId:988(1372ns) IssueIncomingHandshakeReply:193(8067757ns) {SwitchToState:178(2710ns)}}}
2025-11-11T17:18:40.655126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518476314107025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35435
2025-11-11T17:18:40.678098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107025:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.678196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518476314107003:2071] [node 6] ICP18 handshake (actor [3:7571518476314107025:2118]) is too old
2025-11-11T17:18:40.685792Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107002:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.697794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518476314107002:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.702597Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518476314107002:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:40.702859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518472019139663:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.702915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:40.702962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518472019139663:2069] [node 5] ICP22 created new session: [3:7571518476314107027:2069]
2025-11-11T17:18:40.703007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518476314107027:2069] [node 5] ICS09 handshake done sender: [3:7571518476314107002:2110] self: [3:7571518476314107001:0] peer: [5:7571518476599641944:0] socket: 148 qp: -1
2025-11-11T17:18:40.703037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518476314107027:2069] [node 5] ICS10 traffic start
2025-11-11T17:18:40.703071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:40.709901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518476314107030:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35435
2025-11-11T17:18:40.742087Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518476314107016:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:40.774955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518472019139641:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.775023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:40.775062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518472019139641:2049] [node 2] ICP22 created new session: [3:7571518476314107031:2049]
2025-11-11T17:18:40.775118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518476314107031:2049] [node 2] ICS09 handshake done sender: [3:7571518476314107016:2116] self: [3:7571518476314107019:0] peer: [2:7571518476041038269:0] socket: 175 qp: -1
2025-11-11T17:18:40.775147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518476314107031:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:40.775174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:40.805985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107014:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.806029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518476314107014:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.853826Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518476314107014:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:40.850728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518475292805169:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32841
2025-11-11T17:18:40.834851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467805:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.834956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518479369467806:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-11T17:18:40.650118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:40.661820Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038278:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.661861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518476041038278:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.681249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518476041038297:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60210
2025-11-11T17:18:40.692960Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518476041038271:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:40.713986Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518476041038278:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:40.770108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518471746070909:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.770195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:40.770251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518471746070909:2049] [node 3] ICP22 created new session: [2:7571518476041038299:2049]
2025-11-11T17:18:40.770294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518476041038299:2049] [node 3] ICS09 handshake done sender: [2:7571518476041038271:2110] self: [2:7571518476041038269:0] peer: [3:7571518476314107019:0] socket: 170 qp: -1
2025-11-11T17:18:40.770329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518476041038299:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:40.770381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:40.770733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518476041038270:2077] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.770833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:40.770881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518476041038270:2077] [node 5] ICP22 created new session: [2:7571518476041038301:2077]
2025-11-11T17:18:40.770925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518476041038301:2077] [node 5] ICS09 handshake done sender: [2:7571518476041038278:2115] self: [2:7571518476041038277:0] peer: [5:7571518476599641943:0] socket: 172 qp: -1
2025-11-11T17:18:40.770951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518476041038301:2077] [node 5] ICS10 traffic start
2025-11-11T17:18:40.770976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:40.773205Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518476041038276:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:40.773581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518476041038298:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60210
2025-11-11T17:18:40.773736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518471746070908:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.773806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:40.773848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518471746070908:2048] [node 1] ICP22 created new session: [2:7571518476041038304:2048]
2025-11-11T17:18:40.773890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518476041038304:2048] [node 1] ICS09 handshake done sender: [2:7571518476041038276:2114] self: [2:7571518476041038287:0] peer: [1:7571518477136014893:0] socket: 169 qp: -1
2025-11-11T17:18:40.773916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518476041038304:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:40.773949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:40.802384Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:40.806247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518471746070909:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:40.815413Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518476041038289:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.815434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518476041038289:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.846272Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518476041038289:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:40.855930Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518479369467801:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:40.855908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518472019139664:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.856014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:40.856059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518472019139664:2070] [node 4] ICP22 created new session: [3:7571518476314107035:2070]
2025-11-11T17:18:40.856099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518476314107035:2070] [node 4] ICS09 handshake done sender: [3:7571518476314107014:2114] self: [3:7571518476314107012:0] peer: [4:7571518479369467800:0] socket: 188 qp: -1
2025-11-11T17:18:40.856132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518476314107035:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:40.856165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:40.856766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518477136014891:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.856834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:40.856874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518477136014891:2109] [node 6] ICP22 created new session: [1:7571518477136014926:2109]
2025-11-11T17:18:40.856917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518477136014926:2109] [node 6] ICS09 handshake done sender: [1:7571518477136014905:2115] self: [1:7571518477136014904:0] peer: [6:7571518475292805168:0] socket: 181 qp: -1
2025-11-11T17:18:40.856948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518477136014926:2109] [node 6] ICS10 traffic start
2025-11-11T17:18:40.856973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:40.859810Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518475292805165:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:40.860505Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518479369467798:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:40.860805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518475074500411:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.860894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:40.860945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518475074500411:2049] [node 2] ICP22 created new session: [4:7571518479369467807:2049]
2025-11-11T17:18:40.860986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518479369467807:2049] [node 2] ICS09 handshake done sender: [4:7571518479369467801:2120] self: [4:7571518479369467804:0] peer: [2:7571518476041038288:0] socket: 167 qp: -1
2025-11-11T17:18:40.861042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518479369467807:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:40.861090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:40.861382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518475074500412:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.861470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:40.861500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518475074500412:2050] [node 3] ICP22 created new session: [4:7571518479369467809:2050]
2025-11-11T17:18:40.861543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518479369467809:2050] [node 3] ICS09 handshake done sender: [4:7571518479369467798:2118] self: [4:7571518479369467800:0] peer: [3:7571518476314107012:0] socket: 190 qp: -1
2025-11-11T17:18:40.861564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518479369467809:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:40.861623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.867032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518476041038267:2076] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.865819Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:40.877763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518476314107033:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35435
2025-11-11T17:18:40.897861Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518476314107033:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.898025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518472019139640:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:40.879585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518470997837771:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518475292805165:2123] held: no
2025-11-11T17:18:40.900802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518470997837771:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(26894390ns) {HandleHandshakeStatus:395(26891458ns) {LogHandshakeFail:472(6503182ns) {UpdateErrorStateLog:544(6697ns)} DropOutgoingHandshake:456(1648ns) ProcessPendingSessionEvents:495(14454101ns) {DropSessionEvent:512(12411ns) DropSessionEvent:512(5142ns) DropSessionEvent:512(5894ns) DropSessionEvent:512(4502ns) DropSessionEvent:512(5163ns) DropSessionEvent:512(4184ns) DropSessionEvent:512(4879ns) DropSessionEvent:512(4380461ns) DropSessionEvent:512(15594ns) DropSessionEvent:512(5482ns)}}}
2025-11-11T17:18:40.900164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:40.900226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518476041038267:2076] [node 4] ICP22 created new session: [2:7571518476041038307:2076]
2025-11-11T17:18:40.900273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518476041038307:2076] [node 4] ICS09 handshake done sender: [2:7571518476041038289:2118] self: [2:7571518476041038288:0] peer: [4:7571518479369467804:0] socket: 203 qp: -1
2025-11-11T17:18:40.900312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518476041038307:2076] [node 4] ICS10 traffic start
2025-11-11T17:18:40.900380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:40.901207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518476041038267:2076] [node 4] ICP35 event processing took too much time PendingConnection:278(45520957ns) {HandleHandshakeStatus:329(45518289ns) {DropIncomingHandshake:441(3278495ns) DropOutgoingHandshake:456(613ns) SwitchToState:178(313ns) ProcessPendingSessionEvents:495(531846ns) {ForwardSessionEventToSession:612(460738ns) ForwardSessionEventToSession:612(3950ns) ForwardSessionEventToSession:612(5617ns) ForwardSessionEventToSession:612(31652ns) ForwardSessionEventToSession:612(6004ns) ForwardSessionEventToSession:612(4743ns) ForwardSessionEventToSession:612(1927ns) ForwardSessionEventToSession:612(4248ns) ForwardSessionEventToSession:612(1824ns) ForwardSessionEventToSession:612(2116ns)}}}
2025-11-11T17:18:40.902032Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518477136014909:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:40.902067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518477136014909:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:40.903241Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518477136014909:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:40.903518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518472841047532:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:40.903599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:40.903638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518472841047532:2049] [node 3] ICP22 created new session: [1:7571518477136014929:2049]
2025-11-11T17:18:40.903679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518477136014929:2049] [node 3] ICS09 handshake done sender: [1:7571518477136014909:2117] self: [1:7571518477136014908:0] peer: [3:7571518476314107037:0] socket: 142 qp: -1
2025-11-11T17:18:40.903707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518477136014929:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:40.903733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:40.905982Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:40.907255Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518472019139640:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518476314107033:2120] held: no
2025-11-11T17:18:40.907425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518476314107038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35435
2025-11-11T17:18:40.908102Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518476314107033:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:40.908347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518472019139640:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.908408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:40.908453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518472019139640:2048] [node 1] ICP22 created new session: [3:7571518476314107039:2048]
2025-11-11T17:18:40.908490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518476314107039:2048] [node 1] ICS09 handshake done sender: [3:7571518476314107033:2120] self: [3:7571518476314107037:0] peer: [1:7571518477136014908:0] socket: 204 qp: -1
2025-11-11T17:18:40.908518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518476314107039:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:40.908543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:40.908807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518470997837771:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:40.908856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:40.908900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518470997837771:2048] [node 1] ICP22 created new session: [6:7571518475292805170:2048]
2025-11-11T17:18:40.908934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518475292805170:2048] [node 1] ICS09 handshake done sender: [6:7571518475292805165:2123] self: [6:7571518475292805168:0] peer: [1:7571518477136014904:0] socket: 198 qp: -1
2025-11-11T17:18:40.908959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518475292805170:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:40.908979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:40.959145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:40.959387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518476041038304:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-11-11T17:18:40.959426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518471746070908:2048] [node 1] ICP30 unregister session Session# [2:7571518476041038304:2048] VirtualId# [2:7571518476041038287:0]
2025-11-11T17:18:40.959462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518476041038304:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:40.959542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:40.959586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:40.966962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:40.967181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518476314107039:2048] [node 1] ICS01 socket: 204 reason# EndOfStream
2025-11-11T17:18:40.967220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518472019139640:2048] [node 1] ICP30 unregister session Session# [3:7571518476314107039:2048] VirtualId# [3:7571518476314107037:0]
2025-11-11T17:18:40.967252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518476314107039:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:40.967329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:40.967367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:40.973836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:40.974078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518476599641941:2048] [node 1] ICS01 socket: 177 reason# EndOfStream
2025-11-11T17:18:40.974117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518472304674550:2048] [node 1] ICP30 unregister session Session# [5:7571518476599641941:2048] VirtualId# [5:7571518476599641937:0]
2025-11-11T17:18:40.974151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518476599641941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:40.974225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:40.974262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:40.974810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:40.981718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:40.981989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518479369467802:2048] [node 1] ICS01 socket: 165 reason# EndOfStream
2025-11-11T17:18:40.982033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518475074500410:2048] [node 1] ICP30 unregister session Session# [4:7571518479369467802:2048] VirtualId# [4:7571518479369467797:0]
2025-11-11T17:18:40.982067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518479369467802:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:40.982221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:40.982280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:41.009918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518475292805170:2048] [node 1] ICS01 socket: 198 reason# EndOfStream
2025-11-11T17:18:41.009984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518470997837771:2048] [node 1] ICP30 unregister session Session# [6:7571518475292805170:2048] VirtualId# [6:7571518475292805168:0]
2025-11-11T17:18:41.010021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518475292805170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.010141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:41.010189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:41.104005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:41.104264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518476314107031:2049] [node 2] ICS01 socket: 175 reason# EndOfStream
2025-11-11T17:18:41.104308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518472019139641:2049] [node 2] ICP30 unregister session Session# [3:7571518476314107031:2049] VirtualId# [3:7571518476314107019:0]
2025-11-11T17:18:41.104340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518476314107031:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.104426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:41.104474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:41.113780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:41.114034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518479369467807:2049] [node 2] ICS01 socket: 167 reason# EndOfStream
2025-11-11T17:18:41.114075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518475074500411:2049] [node 2] ICP30 unregister session Session# [4:7571518479369467807:2049] VirtualId# [4:7571518479369467804:0]
2025-11-11T17:18:41.114104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518479369467807:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.114185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:41.114226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:41.113777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:41.114028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518475292805163:2049] [node 2] ICS07 socket disconnect 146 reason# EndOfStream
2025-11-11T17:18:41.114070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518475292805163:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.114171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:41.114216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518475292805163:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:41.114899Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13035) Couldn't connect to any resolved address
2025-11-11T17:18:41.115091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:41.115289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518476599641949:2049] [node 2] ICS01 socket: 178 reason# EndOfStream
2025-11-11T17:18:41.115320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518472304674551:2049] [node 2] ICP30 unregister session Session# [5:7571518476599641949:2049] VirtualId# [5:7571518476599641943:0]
2025-11-11T17:18:41.115351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518476599641949:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.115409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:41.115446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:41.115981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518470997837772:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13035) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:41.116095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518475292805163:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:41.116119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518470997837772:2049] [node 2] ICP30 unregister session Session# [6:7571518475292805163:2049] VirtualId# [6:7571518475292805161:0]
2025-11-11T17:18:41.116152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:41.116180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518470997837772:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13035) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:41.116214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13035) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:41.129705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518470997837772:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:41.197757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:41.197975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518476599641947:2050] [node 3] ICS01 socket: 151 reason# EndOfStream
2025-11-11T17:18:41.198024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518472304674552:2050] [node 3] ICP30 unregister session Session# [5:7571518476599641947:2050] VirtualId# [5:7571518476599641944:0]
2025-11-11T17:18:41.198056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518476599641947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.198156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:41.198201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:41.197784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:41.198030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518479369467809:2050] [node 3] ICS01 socket: 190 reason# EndOfStream
2025-11-11T17:18:41.198071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518475074500412:2050] [node 3] ICP30 unregister session Session# [4:7571518479369467809:2050] VirtualId# [4:7571518479369467800:0]
2025-11-11T17:18:41.198100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518479369467809:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.198177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:41.198218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:41.198296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:41.198505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518475292805166:2050] [node 3] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:18:41.198534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518470997837773:2050] [node 3] ICP30 unregister session Session# [6:7571518475292805166:2050] VirtualId# [6:7571518475292805156:0]
2025-11-11T17:18:41.198561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518475292805166:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.198604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:41.198638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:41.277897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:41.278222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518475292805158:2066] [node 4] ICS01 socket: 145 reason# EndOfStream
2025-11-11T17:18:41.278292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518475292805132:2066] [node 4] ICP30 unregister session Session# [6:7571518475292805158:2066] VirtualId# [6:7571518475292805150:0]
2025-11-11T17:18:41.278349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518475292805158:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.278445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:41.278510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:41.281733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:41.281964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518476599641938:2066] [node 4] ICS01 socket: 141 reason# EndOfStream
2025-11-11T17:18:41.282001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518476599641915:2066] [node 4] ICP30 unregister session Session# [5:7571518476599641938:2066] VirtualId# [5:7571518476599641925:0]
2025-11-11T17:18:41.282030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518476599641938:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.282136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:41.282176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:41.341741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:41.341996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518475292805152:2061] [node 5] ICS01 socket: 147 reason# EndOfStream
2025-11-11T17:18:41.342034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518475292805128:2061] [node 5] ICP30 unregister session Session# [6:7571518475292805152:2061] VirtualId# [6:7571518475292805144:0]
2025-11-11T17:18:41.342064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518475292805152:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:41.342211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:41.342256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:41.417496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518481949807788:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:41.417532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518481949807789:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:41.471770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518480519090248:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:41.471826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518480519090249:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:41.513902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518481049575677:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:41.513941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518481049575678:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:41.585889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518483703040130:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:41.585932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518483703040131:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:41.585957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518483703040132:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:41.653027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518479733239530:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:41.653066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518479733239531:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:41.653091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518479733239532:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:41.763546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518480519090263:2061] [node 6] ICP01 ready to work
2025-11-11T17:18:41.750718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518481949807803:2061] [node 6] ICP01 ready to work
2025-11-11T17:18:41.764455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518481049575695:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:41.779538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518480519090266:2063] [node 5] ICP01 ready to work
2025-11-11T17:18:41.842101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518483703040144:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:41.859112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518481049575703:2061] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:41.886419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518479997306029:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:41.886464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518479997306030:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:41.886525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518479997306031:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:41.887160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518479997306045:2060] [node 5] ICP01 ready to work
2025-11-11T17:18:41.887436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518479997306046:2062] [node 4] ICP01 ready to work
2025-11-11T17:18:41.864054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518479733239543:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:41.870846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518481949807815:2070] [node 5] ICP01 ready to work
2025-11-11T17:18:42.018351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518484028206862:2076] [node 4] ICP01 ready to work
2025-11-11T17:18:41.878883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518483703040154:2065] [node 5] ICP01 ready to work
2025-11-11T17:18:41.997543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518481049575732:2062] [node 6] ICP01 ready to work
2025-11-11T17:18:41.877929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518481949807806:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:41.959452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518483703040176:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.146826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518481949807803:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(395749689ns) {Configure:104(395744722ns) {ProcessConfigured:135(44841047ns) {StartInitialHandshake:157(44837610ns) {DropHandshakes:469(4935ns) {DropIncomingHandshake:441(444ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(837ns) PrepareNewSessionHandshake:476(6756ns) {SwitchToState:178(3400ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.157624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206900:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.124930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518485344543030:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65344
2025-11-11T17:18:42.090709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518479997306045:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(201609901ns) {Configure:104(201606159ns) {ProcessConfigured:135(28757307ns) {StartInitialHandshake:157(28754881ns) {DropHandshakes:469(2757ns) {DropIncomingHandshake:441(543ns) DropOutgoingHandshake:456(439ns)} GenerateSessionVirtualId:988(1298ns) PrepareNewSessionHandshake:476(3928ns) {SwitchToState:178(2106ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:41.959523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518483703040177:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.155812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518480519090248:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(384925546ns) {Configure:104(384918730ns) {ProcessConfigured:135(72840ns) {StartInitialHandshake:157(70698ns) {DropHandshakes:469(2112ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(351ns)} GenerateSessionVirtualId:988(997ns) PrepareNewSessionHandshake:476(1863ns) {SwitchToState:178(733ns)}}}}}
2025-11-11T17:18:42.206222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518480519090266:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(376064145ns) {Configure:104(376059420ns) {ProcessConfigured:135(59964ns) {StartInitialHandshake:157(57847ns) {DropHandshakes:469(2701ns) {DropIncomingHandshake:441(584ns) DropOutgoingHandshake:456(349ns)} GenerateSessionVirtualId:988(1014ns) PrepareNewSessionHandshake:476(1461ns) {SwitchToState:178(518ns)}}}}}
2025-11-11T17:18:42.174229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518479997306031:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(149108727ns) {Configure:104(149104776ns) {ProcessConfigured:135(24173632ns) {StartInitialHandshake:157(24170932ns) {DropHandshakes:469(1765ns) {DropIncomingHandshake:441(506ns) DropOutgoingHandshake:456(329ns)} GenerateSessionVirtualId:988(688ns) PrepareNewSessionHandshake:476(4237ns) {SwitchToState:178(1746ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.181658Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17700) Couldn't connect to any resolved address
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.191019Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518484028206900:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.146998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518481949807814:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63335
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.134598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518485344543030:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.234143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518484028206862:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.230315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273403:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
2025-11-11T17:18:42.183399Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518481049575697:2065] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.182225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518483703040176:2085] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.149166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518486244775155:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63335
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.238654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518484814057610:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52553
2025-11-11T17:18:42.214221Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518481949807814:2071] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.254682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518481949807789:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(460619923ns) {Configure:104(460615215ns) {ProcessConfigured:135(67390ns) {StartInitialHandshake:157(64390ns) {DropHandshakes:469(2478ns) {DropIncomingHandshake:441(702ns) DropOutgoingHandshake:456(373ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:476(1955ns) {SwitchToState:178(899ns)}}}}}
2025-11-11T17:18:42.255464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518486244775161:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63335
2025-11-11T17:18:42.267364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518486244775163:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63335
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.183422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518481049575697:2065] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.230495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273404:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.238796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518484814057611:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52553
2025-11-11T17:18:42.237945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518479733239530:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(294869895ns) {Configure:104(294865849ns) {ProcessConfigured:135(16061762ns) {StartInitialHandshake:157(16058269ns) {DropHandshakes:469(3933306ns) {DropIncomingHandshake:441(642ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:476(3951ns) {SwitchToState:178(1434ns)}}}}}
2025-11-11T17:18:42.182322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518483703040144:2059] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17700) Couldn't connect to any resolved address incoming: [0:0:0] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.265196Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518484292273403:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.184088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518485344543045:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65344
2025-11-11T17:18:42.279366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206906:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.278054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775152:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.182428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518483703040144:2059] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:42.313956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518484028206906:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.321914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206907:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.325772Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11965) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.184270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518481049575697:2065] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:42.184474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518481049575695:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.184519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:42.184564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518481049575695:2064] [node 4] ICP22 created new session: [3:7571518485344543047:2064]
2025-11-11T17:18:42.184602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518485344543047:2064] [node 4] ICS09 handshake done sender: [3:7571518481049575697:2065] self: [3:7571518481049575696:0] peer: [4:7571518487998007504:0] socket: 140 qp: -1
2025-11-11T17:18:42.184629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518485344543047:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:42.184663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:42.258494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518481049575678:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(193028528ns) {Configure:104(193022521ns) {ProcessConfigured:135(41365ns) {StartInitialHandshake:157(39205ns) {DropHandshakes:469(2228ns) {DropIncomingHandshake:441(400ns) DropOutgoingHandshake:456(457ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:476(1288ns) {SwitchToState:178(514ns)}}}}}
2025-11-11T17:18:42.259364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518485344543054:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65344
2025-11-11T17:18:42.242605Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057610:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.326567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518479733239531:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.278089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518486244775152:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.182442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:42.182661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518483703040132:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.190155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518487998007506:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
2025-11-11T17:18:42.190222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518487998007507:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
2025-11-11T17:18:42.209691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518483703040144:2059] [node 6] ICP33 wake up from error state
2025-11-11T17:18:42.241991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518483703040156:2066] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.242017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518483703040156:2066] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.242540Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518483703040176:2085] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:42.250116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518487998007505:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
2025-11-11T17:18:42.253775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:308: Proxy [4:7571518483703040144:2059] [node 6] ICP35 event processing took too much time HoldByError:308(16046343ns) {WakeupFromErrorState:799(16044192ns) {SwitchToInitialState:201(4902ns) {SwitchToState:178(3548ns)}}}
2025-11-11T17:18:42.261977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518483703040156:2066] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:42.262513Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518487998007506:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.262557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518483703040132:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.262616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:42.262661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518483703040132:2050] [node 3] ICP22 created new session: [4:7571518487998007510:2050]
2025-11-11T17:18:42.262706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518487998007510:2050] [node 3] ICS09 handshake done sender: [4:7571518483703040176:2085] self: [4:7571518487998007504:0] peer: [3:7571518481049575696:0] socket: 149 qp: -1
2025-11-11T17:18:42.262737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518487998007510:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:42.262781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:42.264196Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518487998007505:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.289813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518483703040154:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.289869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:42.289923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518483703040154:2065] [node 5] ICP22 created new session: [4:7571518487998007512:2065]
2025-11-11T17:18:42.289978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518487998007512:2065] [node 5] ICS09 handshake done sender: [4:7571518483703040156:2066] self: [4:7571518483703040155:0] peer: [5:7571518484028206904:0] socket: 144 qp: -1
2025-11-11T17:18:42.290013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518487998007512:2065] [node 5] ICS10 traffic start
2025-11-11T17:18:42.290045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:42.290554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518483703040130:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.290733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518483703040154:2065] [node 5] ICP18 handshake (actor [4:7571518487998007505:2116]) is too old
2025-11-11T17:18:42.317752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518483703040177:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.319687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518483703040144:2059] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.273820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518479997306031:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.269970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518485344543045:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.273975Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518484292273404:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.274155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518479997306029:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.279660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273407:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.265663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518484814057617:2072] [node 4] ICP01 ready to work
2025-11-11T17:18:42.305277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518481049575677:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(458785740ns) {Configure:104(458780958ns) {ProcessConfigured:135(61780331ns) {StartInitialHandshake:157(61777783ns) {DropHandshakes:469(1780ns) {DropIncomingHandshake:441(231ns) DropOutgoingHandshake:456(252ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:476(2537ns) {SwitchToState:178(392ns)}}}}}
2025-11-11T17:18:42.313343Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518485344543054:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.313556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518485344543057:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65344
2025-11-11T17:18:42.313695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518481049575677:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.324238Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518485344543039:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.324279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518485344543039:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.330843Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518485344543057:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.331107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518481049575678:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.339619Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518485344543039:2107] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:42.357700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518481049575695:2064] [node 4] ICP35 event processing took too much time PendingConnection:278(83930712ns) {HandleHandshakeStatus:329(83928744ns) {DropIncomingHandshake:441(371ns) DropOutgoingHandshake:456(455ns) SwitchToState:178(340ns) ProcessPendingSessionEvents:495(83634922ns) {ForwardSessionEventToSession:612(48989296ns) ForwardSessionEventToSession:612(7753ns) ForwardSessionEventToSession:612(2326ns) ForwardSessionEventToSession:612(14193060ns) ForwardSessionEventToSession:612(4006ns) ForwardSessionEventToSession:612(2621ns) ForwardSessionEventToSession:612(11898428ns) ForwardSessionEventToSession:612(7913ns) ForwardSessionEventToSession:612(8494491ns) ForwardSessionEventToSession:612(7012ns)}}}
2025-11-11T17:18:42.379087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518481049575732:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.379147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:42.379191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518481049575732:2062] [node 6] ICP22 created new session: [3:7571518485344543063:2062]
2025-11-11T17:18:42.379238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518485344543063:2062] [node 6] ICS09 handshake done sender: [3:7571518485344543039:2107] self: [3:7571518485344543038:0] peer: [6:7571518484292273405:0] socket: 158 qp: -1
2025-11-11T17:18:42.379266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518485344543063:2062] [node 6] ICS10 traffic start
2025-11-11T17:18:42.381897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518481049575703:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(437762013ns) {Configure:104(437751713ns) {ProcessConfigured:135(17449571ns) {StartInitialHandshake:157(17447377ns) {DropHandshakes:469(1705ns) {DropIncomingHandshake:441(255ns) DropOutgoingHandshake:456(336ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(1433ns) {SwitchToState:178(114ns)}}}}}
2025-11-11T17:18:42.293817Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518486244775152:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.339884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518484028206862:2076] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11965) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518484028206900:2112] held: no
2025-11-11T17:18:42.359761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518483703040131:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(325758574ns) {Configure:104(325751135ns) {ProcessConfigured:135(9468373ns) {StartInitialHandshake:157(9465592ns) {DropHandshakes:469(1889ns) {DropIncomingHandshake:441(338ns) DropOutgoingHandshake:456(399ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:476(2325ns) {SwitchToState:178(907ns)}}}}}
2025-11-11T17:18:42.393721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:42.420312Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.425214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518481049575678:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518485344543057:2117] held: no
2025-11-11T17:18:42.310180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518481949807788:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(438913528ns) {Configure:104(438905554ns) {ProcessConfigured:135(4099719ns) {StartInitialHandshake:157(4097203ns) {DropHandshakes:469(2090ns) {DropIncomingHandshake:441(495ns) DropOutgoingHandshake:456(406ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(2468ns) {SwitchToState:178(1271ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.382691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206909:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.300831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518484814057619:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52553
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.434138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518485344543065:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65344
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.401950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518487998007520:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
2025-11-11T17:18:42.389836Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518484292273404:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:42.402406Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518479997306050:2064] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.402442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [6:7571518479997306050:2064] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.418002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518479997306029:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.430468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273408:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
2025-11-11T17:18:42.449941Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518484292273403:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.450159Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518485344543057:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:42.451296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518481049575678:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.451360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:42.451403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518481049575678:2049] [node 2] ICP22 created new session: [3:7571518485344543069:2049]
2025-11-11T17:18:42.451439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518485344543069:2049] [node 2] ICS09 handshake done sender: [3:7571518485344543057:2117] self: [3:7571518485344543061:0] peer: [2:7571518484814057618:0] socket: 187 qp: -1
2025-11-11T17:18:42.451467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518485344543069:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:42.451496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:42.301104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518480519090249:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(144888783ns) {Configure:104(144883111ns) {ProcessConfigured:135(23718372ns) {StartInitialHandshake:157(23715700ns) {DropHandshakes:469(5707860ns) {DropIncomingHandshake:441(1721ns) DropOutgoingHandshake:456(719ns)} GenerateSessionVirtualId:988(2596ns) PrepareNewSessionHandshake:476(2322ns) {SwitchToState:178(544ns)}}}}}
2025-11-11T17:18:42.310356Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775154:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.453399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518479997306031:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.453449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:42.453484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518479997306031:2050] [node 3] ICP22 created new session: [6:7571518484292273412:2050]
2025-11-11T17:18:42.453525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518484292273412:2050] [node 3] ICS09 handshake done sender: [6:7571518484292273403:2115] self: [6:7571518484292273405:0] peer: [3:7571518485344543038:0] socket: 170 qp: -1
2025-11-11T17:18:42.453545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518484292273412:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:42.453567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:42.457894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518479997306050:2064] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.472895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7571518479997306046:2062] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.423570Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518484028206906:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:42.417634Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.485685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518485344543066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65344
2025-11-11T17:18:42.306377Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057611:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.310376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518486244775154:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.429883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518487998007516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.473026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:42.509867Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518485344543062:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.509913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518485344543062:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.510573Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.512402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518481049575677:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518485344543054:2116] held: no
2025-11-11T17:18:42.512849Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518485344543062:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:42.318272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518484814057617:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33615008ns) {Configure:104(33611051ns) {ProcessConfigured:135(59521ns) {StartInitialHandshake:157(57993ns) {DropHandshakes:469(1878ns) {DropIncomingHandshake:441(369ns) DropOutgoingHandshake:456(393ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:476(1729ns) {SwitchToState:178(607ns)}}}}}
2025-11-11T17:18:42.434468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518479733239531:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.473070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518479997306046:2062] [node 4] ICP22 created new session: [6:7571518484292273415:2062]
2025-11-11T17:18:42.473109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518484292273415:2062] [node 4] ICS09 handshake done sender: [6:7571518479997306050:2064] self: [6:7571518479997306049:0] peer: [4:7571518487998007515:0] socket: 152 qp: -1
2025-11-11T17:18:42.473143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518484292273415:2062] [node 4] ICS10 traffic start
2025-11-11T17:18:42.473175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:42.473628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273411:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
2025-11-11T17:18:42.474066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273417:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
2025-11-11T17:18:42.477628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:42.485716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518479997306029:2048] [node 1] ICP22 created new session: [6:7571518484292273414:2048]
2025-11-11T17:18:42.505826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518484292273411:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.505960Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518484292273417:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.506095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518479997306045:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.506159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518479997306030:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.515167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518484292273414:2048] [node 1] ICS09 handshake done sender: [6:7571518484292273404:2116] self: [6:7571518484292273406:0] peer: [1:7571518486244775148:0] socket: 171 qp: -1
2025-11-11T17:18:42.515210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518484292273414:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:42.515259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.325581Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518486244775154:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:42.513093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518481049575703:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.434531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.549270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273421:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.559275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518484292273422:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9285
2025-11-11T17:18:42.559681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518484292273417:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:42.559932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518479997306030:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.559982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:42.560019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518479997306030:2049] [node 2] ICP22 created new session: [6:7571518484292273423:2049]
2025-11-11T17:18:42.560063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518484292273423:2049] [node 2] ICS09 handshake done sender: [6:7571518484292273417:2120] self: [6:7571518484292273419:0] peer: [2:7571518484814057623:0] socket: 206 qp: -1
2025-11-11T17:18:42.560097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518484292273423:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:42.560151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:42.331611Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775161:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.437619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518483703040130:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518487998007506:2117] held: no
2025-11-11T17:18:42.326676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057619:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:42.434571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518479733239531:2049] [node 2] ICP22 created new session: [5:7571518484028206912:2049]
2025-11-11T17:18:42.447351Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518487998007520:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.349885Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057621:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.513170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:42.337956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775163:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.574417Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518484292273411:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:42.513210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518481049575703:2061] [node 5] ICP22 created new session: [3:7571518485344543072:2061]
2025-11-11T17:18:42.434611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518484028206912:2049] [node 2] ICS09 handshake done sender: [5:7571518484028206906:2116] self: [5:7571518484028206908:0] peer: [2:7571518484814057613:0] socket: 174 qp: -1
2025-11-11T17:18:42.338033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518481949807803:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.434639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518484028206912:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:42.447498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518483703040131:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.349917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518484814057621:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.338091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:42.434681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:42.513247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518485344543072:2061] [node 5] ICS09 handshake done sender: [3:7571518485344543062:2118] self: [3:7571518485344543059:0] peer: [5:7571518484028206918:0] socket: 203 qp: -1
2025-11-11T17:18:42.574733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518479997306045:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.360431Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518484814057621:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:42.338142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518481949807803:2061] [node 6] ICP22 created new session: [1:7571518486244775168:2061]
2025-11-11T17:18:42.435091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206910:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.503541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518483703040130:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(27754131ns) {HandleHandshakeStatus:395(27748934ns) {LogHandshakeFail:472(14853ns) {UpdateErrorStateLog:544(5466ns)} DropOutgoingHandshake:456(2734ns) ProcessPendingSessionEvents:495(8020715ns) {DropSessionEvent:512(12462ns) DropSessionEvent:512(5016ns) DropSessionEvent:512(3909ns) DropSessionEvent:512(4249ns) DropSessionEvent:512(3572ns) DropSessionEvent:512(4595ns) DropSessionEvent:512(24357ns) DropSessionEvent:512(6291ns) DropSessionEvent:512(4182ns) DropSessionEvent:512(3242ns)}}}
2025-11-11T17:18:42.360975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518480519090249:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.361053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:42.361094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518480519090249:2049] [node 3] ICP22 created new session: [2:7571518484814057631:2049]
2025-11-11T17:18:42.361136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518484814057631:2049] [node 3] ICS09 handshake done sender: [2:7571518484814057621:2115] self: [2:7571518484814057618:0] peer: [3:7571518485344543061:0] socket: 185 qp: -1
2025-11-11T17:18:42.361176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518484814057631:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:42.361204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:42.361916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057614:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.361949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518484814057614:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.376331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518484814057629:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52553
2025-11-11T17:18:42.376395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518484814057630:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52553
2025-11-11T17:18:42.382195Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518484814057614:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:42.421750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057629:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.423060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518480519090248:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.431052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518480519090263:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(509632583ns) {Configure:104(509612050ns) {ProcessConfigured:135(17316892ns) {StartInitialHandshake:157(17314785ns) {DropHandshakes:469(2468ns) {DropIncomingHandshake:441(500ns) DropOutgoingHandshake:456(511ns)} GenerateSessionVirtualId:988(598ns) PrepareNewSessionHandshake:476(2398ns) {SwitchToState:178(766ns)}}}}}
2025-11-11T17:18:42.441229Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057630:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.441292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518480519090266:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.454045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:42.485660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518480519090266:2063] [node 5] ICP22 created new session: [2:7571518484814057636:2063]
2025-11-11T17:18:42.493669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518484814057636:2063] [node 5] ICS09 handshake done sender: [2:7571518484814057614:2113] self: [2:7571518484814057613:0] peer: [5:7571518484028206908:0] socket: 173 qp: -1
2025-11-11T17:18:42.493693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057625:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.493732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518484814057625:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.497681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518484814057636:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:42.497740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:42.519034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518484814057641:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52553
2025-11-11T17:18:42.533846Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518484814057626:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.533887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518484814057626:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.546178Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.547531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518480519090248:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518484814057629:2118] held: no
2025-11-11T17:18:42.547699Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518484814057629:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:42.548198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518480519090248:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.548236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:42.548274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518480519090248:2048] [node 1] ICP22 created new session: [2:7571518484814057644:2048]
2025-11-11T17:18:42.548316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518484814057644:2048] [node 1] ICS09 handshake done sender: [2:7571518484814057629:2118] self: [2:7571518484814057635:0] peer: [1:7571518486244775165:0] socket: 190 qp: -1
2025-11-11T17:18:42.548348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518484814057644:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:42.548381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:42.549122Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518484814057625:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:42.558254Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518484814057626:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:42.597962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518480519090263:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.598031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:42.598084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518480519090263:2061] [node 6] ICP22 created new session: [2:7571518484814057646:2061]
2025-11-11T17:18:42.598134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518484814057646:2061] [node 6] ICS09 handshake done sender: [2:7571518484814057626:2117] self: [2:7571518484814057623:0] peer: [6:7571518484292273419:0] socket: 163 qp: -1
2025-11-11T17:18:42.598169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518484814057646:2061] [node 6] ICS10 traffic start
2025-11-11T17:18:42.598203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:42.619326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518484814057617:2072] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.646481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518480519090266:2063] [node 5] ICP35 event processing took too much time PendingConnection:278(198624806ns) {HandleHandshakeStatus:329(198622558ns) {DropIncomingHandshake:441(13076ns) DropOutgoingHandshake:456(350ns) SwitchToState:178(537ns) ProcessPendingSessionEvents:495(74193290ns) {ForwardSessionEventToSession:612(39941243ns) ForwardSessionEventToSession:612(6189ns) ForwardSessionEventToSession:612(4334ns) ForwardSessionEventToSession:612(7970913ns) ForwardSessionEventToSession:612(4040ns) ForwardSessionEventToSession:612(2438ns) ForwardSessionEventToSession:612(6884562ns) ForwardSessionEventToSession:612(14717096ns) ForwardSessionEventToSession:612(4629321ns) ForwardSessionEventToSession:612(6056ns)}}}
2025-11-11T17:18:42.647674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:42.647726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518484814057617:2072] [node 4] ICP22 created new session: [2:7571518484814057648:2072]
2025-11-11T17:18:42.647772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518484814057648:2072] [node 4] ICS09 handshake done sender: [2:7571518484814057625:2116] self: [2:7571518484814057624:0] peer: [4:7571518487998007521:0] socket: 165 qp: -1
2025-11-11T17:18:42.647822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518484814057648:2072] [node 4] ICS10 traffic start
2025-11-11T17:18:42.647874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:42.574795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:42.513279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518485344543072:2061] [node 5] ICS10 traffic start
2025-11-11T17:18:42.338186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518486244775168:2061] [node 6] ICS09 handshake done sender: [1:7571518486244775152:2111] self: [1:7571518486244775148:0] peer: [6:7571518484292273406:0] socket: 161 qp: -1
2025-11-11T17:18:42.574841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518479997306045:2060] [node 5] ICP22 created new session: [6:7571518484292273425:2060]
2025-11-11T17:18:42.574886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518484292273425:2060] [node 5] ICS09 handshake done sender: [6:7571518484292273411:2119] self: [6:7571518484292273418:0] peer: [5:7571518479733239544:0] socket: 142 qp: -1
2025-11-11T17:18:42.574919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518484292273425:2060] [node 5] ICS10 traffic start
2025-11-11T17:18:42.574970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:42.593968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518479997306029:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(155887039ns) {HandleHandshakeStatus:329(155884159ns) {DropIncomingHandshake:441(991ns) DropOutgoingHandshake:456(4987ns) SwitchToState:178(720ns) ProcessPendingSessionEvents:495(8012134ns) {ForwardSessionEventToSession:612(7964448ns) ForwardSessionEventToSession:612(4619ns) ForwardSessionEventToSession:612(5286ns) ForwardSessionEventToSession:612(5627ns) ForwardSessionEventToSession:612(2536ns) ForwardSessionEventToSession:612(4902ns) ForwardSessionEventToSession:612(3405ns) ForwardSessionEventToSession:612(3247ns) ForwardSessionEventToSession:612(5346ns) ForwardSessionEventToSession:612(2806ns)}}}
2025-11-11T17:18:42.634009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518479997306031:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(68885245ns) {HandleHandshakeStatus:329(68884039ns) {DropIncomingHandshake:441(390ns) DropOutgoingHandshake:456(3631ns) SwitchToState:178(260ns) ProcessPendingSessionEvents:495(68077374ns) {ForwardSessionEventToSession:612(35504007ns) ForwardSessionEventToSession:612(4944ns) ForwardSessionEventToSession:612(2457ns) ForwardSessionEventToSession:612(4979865ns) ForwardSessionEventToSession:612(11014413ns) ForwardSessionEventToSession:612(6662ns) ForwardSessionEventToSession:612(8210668ns) ForwardSessionEventToSession:612(2667ns) ForwardSessionEventToSession:612(8328192ns) ForwardSessionEventToSession:612(3499ns)}}}
2025-11-11T17:18:42.450763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206914:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.450805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206915:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.458560Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518484028206910:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.469792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518484028206862:2076] [node 4] ICP35 event processing took too much time PendingConnection:278(31190216ns) {HandleHandshakeStatus:395(31186947ns) {LogHandshakeFail:472(16523ns) {UpdateErrorStateLog:544(5016ns)} DropOutgoingHandshake:456(1364ns) ProcessPendingSessionEvents:495(17895227ns) {DropSessionEvent:512(17031ns) DropSessionEvent:512(4232ns) DropSessionEvent:512(3525ns) DropSessionEvent:512(5291ns) DropSessionEvent:512(3594ns) DropSessionEvent:512(3561ns) DropSessionEvent:512(3862ns) DropSessionEvent:512(28260ns) DropSessionEvent:512(4472ns) DropSessionEvent:512(7583875ns)}}}
2025-11-11T17:18:42.470040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518479733239530:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.473758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518484028206915:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.477705Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518484028206900:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:42.486668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518479733239532:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:42.497965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518484028206862:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.498052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:42.498115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518484028206862:2076] [node 4] ICP22 created new session: [5:7571518484028206919:2076]
2025-11-11T17:18:42.498164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518484028206919:2076] [node 4] ICS09 handshake done sender: [5:7571518484028206900:2112] self: [5:7571518484028206904:0] peer: [4:7571518483703040155:0] socket: 159 qp: -1
2025-11-11T17:18:42.498193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518484028206919:2076] [node 4] ICS10 traffic start
2025-11-11T17:18:42.498227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:42.526500Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518484028206914:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.526736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206921:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.537721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518479733239545:2060] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.537764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518479733239545:2060] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.550250Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16639) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.558540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518479733239532:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16639) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518484028206915:2121] held: no
2025-11-11T17:18:42.570260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518484028206922:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58925
2025-11-11T17:18:42.570348Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518479733239545:2060] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:42.613715Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518484028206910:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:42.618256Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.625781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518479733239543:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.625861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:42.625908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518479733239543:2059] [node 6] ICP22 created new session: [5:7571518484028206925:2059]
2025-11-11T17:18:42.625952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518484028206925:2059] [node 6] ICS09 handshake done sender: [5:7571518479733239545:2060] self: [5:7571518479733239544:0] peer: [6:7571518484292273418:0] socket: 202 qp: -1
2025-11-11T17:18:42.625994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518484028206925:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:42.626031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:42.637974Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518484028206915:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:42.638210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518479733239530:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518484028206910:2119] held: no
2025-11-11T17:18:42.638295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518479733239530:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.638361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:42.638408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518479733239530:2048] [node 1] ICP22 created new session: [5:7571518484028206927:2048]
2025-11-11T17:18:42.638451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518484028206927:2048] [node 1] ICS09 handshake done sender: [5:7571518484028206910:2119] self: [5:7571518484028206917:0] peer: [1:7571518486244775173:0] socket: 200 qp: -1
2025-11-11T17:18:42.638484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518484028206927:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:42.638516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:42.639389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518479733239532:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.639440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:42.639489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518479733239532:2050] [node 3] ICP22 created new session: [5:7571518484028206929:2050]
2025-11-11T17:18:42.639528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518484028206929:2050] [node 3] ICS09 handshake done sender: [5:7571518484028206915:2121] self: [5:7571518484028206918:0] peer: [3:7571518485344543059:0] socket: 205 qp: -1
2025-11-11T17:18:42.639555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518484028206929:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:42.639586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:42.513309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:42.525913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518481049575695:2064] [node 4] ICP18 handshake (actor [3:7571518485344543045:2112]) is too old
2025-11-11T17:18:42.526186Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518485344543054:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:42.538479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518481049575677:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.538554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:42.538605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518481049575677:2048] [node 1] ICP22 created new session: [3:7571518485344543074:2048]
2025-11-11T17:18:42.538646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518485344543074:2048] [node 1] ICS09 handshake done sender: [3:7571518485344543054:2116] self: [3:7571518485344543058:0] peer: [1:7571518486244775159:0] socket: 178 qp: -1
2025-11-11T17:18:42.538681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518485344543074:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:42.538715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:42.581694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518481049575732:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(150579246ns) {HandleHandshakeStatus:329(150576616ns) {DropIncomingHandshake:441(335ns) DropOutgoingHandshake:456(581ns) SwitchToState:178(470ns) ProcessPendingSessionEvents:495(71754010ns) {ForwardSessionEventToSession:612(16413049ns) ForwardSessionEventToSession:612(7641743ns) ForwardSessionEventToSession:612(3267ns) ForwardSessionEventToSession:612(2956ns) ForwardSessionEventToSession:612(12794014ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(10968809ns) ForwardSessionEventToSession:612(8160334ns) ForwardSessionEventToSession:612(5662ns) ForwardSessionEventToSession:612(15732317ns)}}}
2025-11-11T17:18:42.710999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518484814057617:2072] [node 4] ICP35 event processing took too much time PendingConnection:278(95538412ns) {HandleHandshakeStatus:329(95535073ns) {DropIncomingHandshake:441(23149ns) DropOutgoingHandshake:456(567ns) SwitchToState:178(417ns) ProcessPendingSessionEvents:495(57791737ns) {ForwardSessionEventToSession:612(17819042ns) ForwardSessionEventToSession:612(7673960ns) ForwardSessionEventToSession:612(6456ns) ForwardSessionEventToSession:612(4230549ns) ForwardSessionEventToSession:612(15944194ns) ForwardSessionEventToSession:612(4535ns) ForwardSessionEventToSession:612(3355ns) ForwardSessionEventToSession:612(363554ns) ForwardSessionEventToSession:612(2562ns) ForwardSessionEventToSession:612(11714208ns)}}}
2025-11-11T17:18:42.517679Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518487998007506:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:42.533657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518487998007522:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
2025-11-11T17:18:42.549363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518483703040130:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.549406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:42.549447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518483703040130:2048] [node 1] ICP22 created new session: [4:7571518487998007523:2048]
2025-11-11T17:18:42.549496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518487998007523:2048] [node 1] ICS09 handshake done sender: [4:7571518487998007506:2117] self: [4:7571518487998007514:0] peer: [1:7571518486244775153:0] socket: 168 qp: -1
2025-11-11T17:18:42.549522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518487998007523:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:42.549549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:42.566574Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518483703040177:2086] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:42.566855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518483703040144:2059] [node 6] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.566919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:42.566971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518483703040144:2059] [node 6] ICP22 created new session: [4:7571518487998007525:2059]
2025-11-11T17:18:42.567032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518487998007525:2059] [node 6] ICS09 handshake done sender: [4:7571518483703040177:2086] self: [4:7571518487998007515:0] peer: [6:7571518479997306049:0] socket: 153 qp: -1
2025-11-11T17:18:42.567062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518487998007525:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:42.567095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:42.569999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518487998007527:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48430
2025-11-11T17:18:42.621688Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518487998007520:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:42.649921Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:42.657963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518483703040131:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518487998007520:2121] held: no
2025-11-11T17:18:42.681857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518483703040131:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31450652ns) {HandleHandshakeStatus:395(31446183ns) {LogHandshakeFail:472(12807ns) {UpdateErrorStateLog:544(4808ns)} DropOutgoingHandshake:456(2043ns) ProcessPendingSessionEvents:495(23671392ns) {DropSessionEvent:512(10584ns) DropSessionEvent:512(3582ns) DropSessionEvent:512(4170558ns) DropSessionEvent:512(11230291ns) DropSessionEvent:512(8210206ns) DropSessionEvent:512(19102ns) DropSessionEvent:512(3878ns) DropSessionEvent:512(3199ns) DropSessionEvent:512(3305ns) DropSessionEvent:512(2725ns)}}}
2025-11-11T17:18:42.681922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518483703040131:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:42.682001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:42.682054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518483703040131:2049] [node 2] ICP22 created new session: [4:7571518487998007528:2049]
2025-11-11T17:18:42.682102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518487998007528:2049] [node 2] ICS09 handshake done sender: [4:7571518487998007520:2121] self: [4:7571518487998007521:0] peer: [2:7571518484814057624:0] socket: 197 qp: -1
2025-11-11T17:18:42.682133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518487998007528:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:42.682167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:42.338219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518486244775168:2061] [node 6] ICS10 traffic start
2025-11-11T17:18:42.338252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:42.349894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518481949807815:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(471670212ns) {Configure:104(471664966ns) {ProcessConfigured:135(10325423ns) {StartInitialHandshake:157(10322729ns) {DropHandshakes:469(4173ns) {DropIncomingHandshake:441(403ns) DropOutgoingHandshake:456(426ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:476(3474ns) {SwitchToState:178(1591ns)}}}}}
2025-11-11T17:18:42.354421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518481949807806:2059] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.354494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:42.354541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518481949807806:2059] [node 4] ICP22 created new session: [1:7571518486244775175:2059]
2025-11-11T17:18:42.354584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518486244775175:2059] [node 4] ICS09 handshake done sender: [1:7571518486244775154:2112] self: [1:7571518486244775153:0] peer: [4:7571518487998007514:0] socket: 162 qp: -1
2025-11-11T17:18:42.354615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518486244775175:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:42.354655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:42.362144Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775160:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.362172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518486244775160:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.417754Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518486244775160:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:42.417851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775155:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.445877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518481949807789:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.445960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:42.446016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518481949807789:2049] [node 3] ICP22 created new session: [1:7571518486244775179:2049]
2025-11-11T17:18:42.446063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518486244775179:2049] [node 3] ICS09 handshake done sender: [1:7571518486244775160:2114] self: [1:7571518486244775159:0] peer: [3:7571518485344543058:0] socket: 177 qp: -1
2025-11-11T17:18:42.446100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518486244775179:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:42.457721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:42.458068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518481949807803:2061] [node 6] ICP18 handshake (actor [1:7571518486244775155:2113]) is too old
2025-11-11T17:18:42.466094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [1:7571518481949807803:2061] [node 6] ICP35 event processing took too much time StateWork:293(16275524ns) {IncomingHandshake:261(16271768ns)}
2025-11-11T17:18:42.477654Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775166:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.477702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518486244775166:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.502780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518486244775181:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63335
2025-11-11T17:18:42.503545Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775174:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.503579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518486244775174:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:42.518536Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518486244775166:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:42.518677Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518486244775174:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:42.519338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518481949807815:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.519403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:42.519454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518481949807815:2070] [node 5] ICP22 created new session: [1:7571518486244775184:2070]
2025-11-11T17:18:42.519489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518486244775184:2070] [node 5] ICS09 handshake done sender: [1:7571518486244775174:2118] self: [1:7571518486244775173:0] peer: [5:7571518484028206917:0] socket: 198 qp: -1
2025-11-11T17:18:42.519521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518486244775184:2070] [node 5] ICS10 traffic start
2025-11-11T17:18:42.519553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:42.520237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518481949807788:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:42.520290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:42.520329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518481949807788:2048] [node 2] ICP22 created new session: [1:7571518486244775186:2048]
2025-11-11T17:18:42.520374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518486244775186:2048] [node 2] ICS09 handshake done sender: [1:7571518486244775166:2117] self: [1:7571518486244775165:0] peer: [2:7571518484814057635:0] socket: 189 qp: -1
2025-11-11T17:18:42.520404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518486244775186:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:42.520432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:42.561875Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518486244775181:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:42.570533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518481949807815:2070] [node 5] ICP18 handshake (actor [1:7571518486244775181:2119]) is too old
2025-11-11T17:18:42.649795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518481949807789:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(99947634ns) {HandleHandshakeStatus:329(99944437ns) {DropIncomingHandshake:441(13391ns) DropOutgoingHandshake:456(425ns) SwitchToState:178(841ns) ProcessPendingSessionEvents:495(60082254ns) {ForwardSessionEventToSession:612(19974009ns) ForwardSessionEventToSession:612(1000407ns) ForwardSessionEventToSession:612(3492ns) ForwardSessionEventToSession:612(7308742ns) ForwardSessionEventToSession:612(8375883ns) ForwardSessionEventToSession:612(2793ns) ForwardSessionEventToSession:612(7490686ns) ForwardSessionEventToSession:612(4184ns) ForwardSessionEventToSession:612(7963366ns) ForwardSessionEventToSession:612(7936506ns)}}}
2025-11-11T17:18:42.787138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:42.787383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518484814057644:2048] [node 1] ICS01 socket: 190 reason# EndOfStream
2025-11-11T17:18:42.787420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518480519090248:2048] [node 1] ICP30 unregister session Session# [2:7571518484814057644:2048] VirtualId# [2:7571518484814057635:0]
2025-11-11T17:18:42.787454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518484814057644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.787517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:42.787549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:42.790928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:42.791125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518485344543074:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-11T17:18:42.791168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518481049575677:2048] [node 1] ICP30 unregister session Session# [3:7571518485344543074:2048] VirtualId# [3:7571518485344543058:0]
2025-11-11T17:18:42.791197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518485344543074:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.791271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:42.791314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:42.809724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:42.809974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518484292273414:2048] [node 1] ICS07 socket disconnect 171 reason# EndOfStream
2025-11-11T17:18:42.810002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518484292273414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.810111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:42.810159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518484292273414:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:42.811047Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Couldn't connect to any resolved address
2025-11-11T17:18:42.811311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518479997306029:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:42.811355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518484292273414:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:42.811385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518479997306029:2048] [node 1] ICP30 unregister session Session# [6:7571518484292273414:2048] VirtualId# [6:7571518484292273406:0]
2025-11-11T17:18:42.811433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:42.811459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518479997306029:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:42.811473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:42.811732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:42.811981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518484028206927:2048] [node 1] ICS01 socket: 200 reason# EndOfStream
2025-11-11T17:18:42.812027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518479733239530:2048] [node 1] ICP30 unregister session Session# [5:7571518484028206927:2048] VirtualId# [5:7571518484028206917:0]
2025-11-11T17:18:42.812052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518484028206927:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.812130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:42.812167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:42.813739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:42.813911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518487998007523:2048] [node 1] ICS01 socket: 168 reason# EndOfStream
2025-11-11T17:18:42.813945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518483703040130:2048] [node 1] ICP30 unregister session Session# [4:7571518487998007523:2048] VirtualId# [4:7571518487998007514:0]
2025-11-11T17:18:42.813970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518487998007523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.814050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:42.814095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:42.821730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518479997306029:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:42.866825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:42.867090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518485344543069:2049] [node 2] ICS01 socket: 187 reason# EndOfStream
2025-11-11T17:18:42.867132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518481049575678:2049] [node 2] ICP30 unregister session Session# [3:7571518485344543069:2049] VirtualId# [3:7571518485344543061:0]
2025-11-11T17:18:42.867166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518485344543069:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.867252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:42.867297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:42.881761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:42.881728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:42.881965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518484028206912:2049] [node 2] ICS01 socket: 174 reason# EndOfStream
2025-11-11T17:18:42.881956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518487998007528:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-11T17:18:42.882003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518479733239531:2049] [node 2] ICP30 unregister session Session# [5:7571518484028206912:2049] VirtualId# [5:7571518484028206908:0]
2025-11-11T17:18:42.881995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518483703040131:2049] [node 2] ICP30 unregister session Session# [4:7571518487998007528:2049] VirtualId# [4:7571518487998007521:0]
2025-11-11T17:18:42.882029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518484028206912:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.882027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518487998007528:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.882104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:42.882104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:42.882143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:42.882142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:42.885764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:42.885999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518484292273423:2049] [node 2] ICS01 socket: 206 reason# EndOfStream
2025-11-11T17:18:42.886045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518479997306030:2049] [node 2] ICP30 unregister session Session# [6:7571518484292273423:2049] VirtualId# [6:7571518484292273419:0]
2025-11-11T17:18:42.886079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518484292273423:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.886453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:42.886521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:42.983767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:42.984003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518484292273412:2050] [node 3] ICS07 socket disconnect 170 reason# EndOfStream
2025-11-11T17:18:42.984030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518484292273412:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.984112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:42.984149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518484292273412:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:42.984649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:42.984840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518484028206929:2050] [node 3] ICS01 socket: 205 reason# EndOfStream
2025-11-11T17:18:42.984877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518479733239532:2050] [node 3] ICP30 unregister session Session# [5:7571518484028206929:2050] VirtualId# [5:7571518484028206918:0]
2025-11-11T17:18:42.984906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518484028206929:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:42.984973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:42.985013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:42.985680Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16639) Couldn't connect to any resolved address
2025-11-11T17:18:43.005932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518479997306031:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16639) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:42.989788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:43.005873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518487998007510:2050] [node 3] ICS01 socket: 149 reason# EndOfStream
2025-11-11T17:18:43.009753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518483703040132:2050] [node 3] ICP30 unregister session Session# [4:7571518487998007510:2050] VirtualId# [4:7571518487998007504:0]
2025-11-11T17:18:43.009810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518487998007510:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:43.009925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:43.009974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:43.014048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518484292273412:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:43.014100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518479997306031:2050] [node 3] ICP30 unregister session Session# [6:7571518484292273412:2050] VirtualId# [6:7571518484292273405:0]
2025-11-11T17:18:43.014145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:43.014178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518479997306031:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16639) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:43.014196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16639) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:43.014426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [6:7571518479997306031:2050] [node 3] ICP35 event processing took too much time StateWork:293(28346343ns) {HandleHandshakeStatus:395(28343125ns) {LogHandshakeFail:472(9916ns) {UpdateErrorStateLog:544(1546ns)} DropOutgoingHandshake:456(2971ns) ProcessPendingSessionEvents:495(287ns) UnregisterSession:538(46002ns) {ProcessPendingSessionEvents:495(197ns) SwitchToInitialState:201(1233ns) {SwitchToState:178(521ns)}} TransitToErrorState:752(52507ns) {SwitchToState:178(5030ns) ProcessPendingSessionEvents:495(760ns) DropHandshakes:469(1838ns) {DropIncomingHandshake:441(332ns) DropOutgoingHandshake:456(300ns)}}}}
2025-11-11T17:18:43.053988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518479997306031:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:43.109789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:43.110119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518484292273415:2062] [node 4] ICS01 socket: 152 reason# EndOfStream
2025-11-11T17:18:43.110169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518479997306046:2062] [node 4] ICP30 unregister session Session# [6:7571518484292273415:2062] VirtualId# [6:7571518479997306049:0]
2025-11-11T17:18:43.110225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518484292273415:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:43.110361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:43.110435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:43.109867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:43.110118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518484028206919:2076] [node 4] ICS01 socket: 159 reason# EndOfStream
2025-11-11T17:18:43.110182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518484028206862:2076] [node 4] ICP30 unregister session Session# [5:7571518484028206919:2076] VirtualId# [5:7571518484028206904:0]
2025-11-11T17:18:43.110233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518484028206919:2076] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:43.110355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:43.110418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:43.381944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518491707265378:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:43.381992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518491707265379:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:43.437790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518488623336779:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:43.437832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518488623336780:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:43.504363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518491412910821:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:43.504415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518491412910822:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:43.552679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518488108330776:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:43.552720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518488108330777:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:43.552746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518488108330778:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:43.637889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518492002253231:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:43.637927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518492002253232:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:43.637949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518492002253233:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:44.125916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518496297220546:2061] [node 4] ICP01 ready to work
2025-11-11T17:18:44.126811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518495707878152:2069] [node 5] ICP01 ready to work
2025-11-11T17:18:44.170542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518492403298102:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.180205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220547:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:43.862021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518489796220638:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:44.203939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518492918304128:2075] [node 4] ICP01 ready to work
2025-11-11T17:18:44.219534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518488623336779:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(93217024ns) {Configure:104(93213009ns) {ProcessConfigured:135(97661ns) {StartInitialHandshake:157(95082ns) {DropHandshakes:469(3107ns) {DropIncomingHandshake:441(560ns) DropOutgoingHandshake:456(569ns)} GenerateSessionVirtualId:988(1201ns) PrepareNewSessionHandshake:476(1267ns) {SwitchToState:178(432ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.222688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220548:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:43.862059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518489796220639:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.318127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518496297220555:2070] [node 6] ICP01 ready to work
2025-11-11T17:18:44.319762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220554:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.339125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518492403298128:2092] [node 6] ICP01 ready to work
2025-11-11T17:18:44.340779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298127:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.269699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518495707878180:2070] [node 6] ICP01 ready to work
2025-11-11T17:18:44.267863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518492918304146:2096] [node 6] ICP01 ready to work
2025-11-11T17:18:44.276891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518496002232736:2094] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:43.862082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518489796220640:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.329744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518495707878184:2068] [node 4] ICP01 ready to work
2025-11-11T17:18:44.388614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518491412910821:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(261525819ns) {Configure:104(261522459ns) {ProcessConfigured:135(18636423ns) {StartInitialHandshake:157(18633735ns) {DropHandshakes:469(2696ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(429ns)} GenerateSessionVirtualId:988(865ns) PrepareNewSessionHandshake:476(4636ns) {SwitchToState:178(2454ns)}}}}}
2025-11-11T17:18:44.328366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518496002232738:2095] [node 6] ICP01 ready to work
2025-11-11T17:18:44.143996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518494091187947:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:44.274218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518492918304148:2076] [node 5] ICP01 ready to work
2025-11-11T17:18:44.434834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518488108330778:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(248184023ns) {Configure:104(248177397ns) {ProcessConfigured:135(62592ns) {StartInitialHandshake:157(60345ns) {DropHandshakes:469(2128ns) {DropIncomingHandshake:441(539ns) DropOutgoingHandshake:456(427ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:476(1489ns) {SwitchToState:178(536ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.448481Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518492403298127:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.458315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298147:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.442329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518495707878189:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48670
2025-11-11T17:18:44.376345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518496002232740:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14578
2025-11-11T17:18:44.183708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518494091187952:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:44.454712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518496297220546:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(328458131ns) {Configure:104(328444739ns) {ProcessConfigured:135(52968ns) {StartInitialHandshake:157(50683ns) {DropHandshakes:469(2064ns) {DropIncomingHandshake:441(483ns) DropOutgoingHandshake:456(385ns)} GenerateSessionVirtualId:988(859ns) PrepareNewSessionHandshake:476(1534ns) {SwitchToState:178(502ns)}}}}}
2025-11-11T17:18:44.376414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518496002232741:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14578
2025-11-11T17:18:44.458963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878189:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.456904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220598:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:44.377650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091187966:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.510849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518495707878152:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(308947057ns) {Configure:104(298068325ns) {ProcessConfigured:135(64603ns) {StartInitialHandshake:157(61809ns) {DropHandshakes:469(2200ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(310ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:476(826ns) {SwitchToState:178(273ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.389640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518496002232744:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14578
2025-11-11T17:18:44.519817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518492403298104:2077] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.377699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091187967:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.461691Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518496297220554:2073] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.377948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518496002232739:2096] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.590702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518495707878184:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(161557111ns) {Configure:104(161546127ns) {ProcessConfigured:135(80885ns) {StartInitialHandshake:157(75878ns) {DropHandshakes:469(497ns) {DropIncomingHandshake:441(144ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(1460ns) {SwitchToState:178(539ns)}}}}}
2025-11-11T17:18:44.519859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518492403298104:2077] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.549689Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304150:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.486055Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518494091187966:2070] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.462250Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518496297220547:2066] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.525991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518492403298130:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.486750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518494091187967:2071] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.549752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518492918304150:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.417851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232740:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.486243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518496297220546:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.573930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518492918304152:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-11-11T17:18:44.526033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518492403298130:2100] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.486832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518494091187947:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.486388Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518496297220548:2067] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.497662Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232741:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.658050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518495707878197:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48670
2025-11-11T17:18:44.526587Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518492403298147:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.622811Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304152:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.509708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518494091187952:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.518144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518492002253232:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(436520799ns) {Configure:104(436515552ns) {ProcessConfigured:135(27699150ns) {StartInitialHandshake:157(27696511ns) {DropHandshakes:469(2066ns) {DropIncomingHandshake:441(314ns) DropOutgoingHandshake:456(507ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:476(3428ns) {SwitchToState:178(1352ns)}}}}}
2025-11-11T17:18:44.689062Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878191:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.623210Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518492918304150:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.518308Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518496297220557:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.503268Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518496002232739:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(40727497ns) {Configure:104(40722314ns) {ProcessConfigured:135(85247ns) {StartInitialHandshake:157(80741ns) {DropHandshakes:469(1980ns) {DropIncomingHandshake:441(385ns) DropOutgoingHandshake:456(369ns)} GenerateSessionVirtualId:988(631ns) PrepareNewSessionHandshake:476(1455ns) {SwitchToState:178(442ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.546158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518492403298130:2100] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:44.623443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518492918304148:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.530133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518494091187952:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(21954463ns) {IncomingHandshake:261(21952805ns) {DropIncomingHandshake:441(158ns) PrepareNewSessionHandshake:476(784ns) {SwitchToState:178(451ns)} GenerateSessionVirtualId:988(236ns) IssueIncomingHandshakeReply:193(68449ns) {SwitchToState:178(497ns)}}}
2025-11-11T17:18:44.515442Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232744:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.689080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518495707878191:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.623487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:44.623539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518492918304148:2076] [node 5] ICP22 created new session: [2:7571518492918304155:2076]
2025-11-11T17:18:44.623577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518492918304155:2076] [node 5] ICS09 handshake done sender: [2:7571518492918304150:2116] self: [2:7571518492918304149:0] peer: [5:7571518496297220606:0] socket: 156 qp: -1
2025-11-11T17:18:44.623603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518492918304155:2076] [node 5] ICS10 traffic start
2025-11-11T17:18:44.623630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:44.643669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518492918304128:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(385232394ns) {Configure:104(385220748ns) {ProcessConfigured:135(17842795ns) {StartInitialHandshake:157(17840161ns) {DropHandshakes:469(1535ns) {DropIncomingHandshake:441(289ns) DropOutgoingHandshake:456(253ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:476(2529ns) {SwitchToState:178(1178ns)}}}}}
2025-11-11T17:18:44.717649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518492918304162:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-11-11T17:18:44.741681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518492918304146:2096] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(343908127ns) {Configure:104(327507027ns) {ProcessConfigured:135(51983467ns) {StartInitialHandshake:157(51979316ns) {DropHandshakes:469(2746ns) {DropIncomingHandshake:441(746ns) DropOutgoingHandshake:456(373ns)} GenerateSessionVirtualId:988(693ns) PrepareNewSessionHandshake:476(23440545ns) {SwitchToState:178(2119ns)}}}}}
2025-11-11T17:18:44.741824Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304162:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.741903Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304158:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.741923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518492918304158:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.756977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518492918304166:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-11-11T17:18:44.757074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518492918304167:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-11-11T17:18:44.787337Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304166:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.518332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518496297220557:2074] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.581145Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518492403298128:2092] [node 6] ICP35 event processing took too much time PendingConnection:278(34409971ns) {IncomingHandshake:261(16030742ns) {DropIncomingHandshake:441(578ns) PrepareNewSessionHandshake:476(1763ns) {SwitchToState:178(713ns)} GenerateSessionVirtualId:988(648ns) IssueIncomingHandshakeReply:193(11509ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.797773Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518492918304158:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:44.804691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518488623336779:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.689776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878197:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.566661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518496002232738:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(137465143ns) {Configure:104(137456944ns) {ProcessConfigured:135(62627ns) {StartInitialHandshake:157(60198ns) {DropHandshakes:469(2512ns) {DropIncomingHandshake:441(615ns) DropOutgoingHandshake:456(691ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:476(1577ns) {SwitchToState:178(748ns)}}}}}
2025-11-11T17:18:44.594359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518492403298104:2077] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:44.518964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518492002253232:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.821680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518488623336780:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(492738891ns) {Configure:104(492733903ns) {ProcessConfigured:135(40312871ns) {StartInitialHandshake:157(40309872ns) {DropHandshakes:469(2713ns) {DropIncomingHandshake:441(611ns) DropOutgoingHandshake:456(558ns)} GenerateSessionVirtualId:988(841ns) PrepareNewSessionHandshake:476(3045ns) {SwitchToState:178(951ns)}}}}}
2025-11-11T17:18:44.831087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304167:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.832957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518492918304171:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-11-11T17:18:44.846891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518492918304128:2075] [node 4] ICP35 event processing took too much time PendingConnection:278(23984794ns) {IncomingHandshake:261(3976355ns) {DropIncomingHandshake:441(394ns) PrepareNewSessionHandshake:476(1883ns) {SwitchToState:178(744ns)} GenerateSessionVirtualId:988(618ns) IssueIncomingHandshakeReply:193(8044ns)}}
2025-11-11T17:18:44.851157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518492918304128:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.567513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188016:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.690527Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518495707878191:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:44.591494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518491707265378:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(378220925ns) {Configure:104(378214396ns) {ProcessConfigured:135(13377512ns) {StartInitialHandshake:157(13374846ns) {DropHandshakes:469(2787ns) {DropIncomingHandshake:441(810ns) DropOutgoingHandshake:456(461ns)} GenerateSessionVirtualId:988(729ns) PrepareNewSessionHandshake:476(2482ns) {SwitchToState:178(101ns)}}}}}
2025-11-11T17:18:44.857773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518488623336779:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23885341ns) {IncomingHandshake:261(23882796ns) {DropIncomingHandshake:441(377ns) PrepareNewSessionHandshake:476(3213ns) {SwitchToState:178(2260ns)} GenerateSessionVirtualId:988(515ns) IssueIncomingHandshakeReply:193(9084043ns) {SwitchToState:178(4129ns)}}}
2025-11-11T17:18:44.601888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518488108330776:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(402909884ns) {Configure:104(402907755ns) {ProcessConfigured:135(8874117ns) {StartInitialHandshake:157(8871361ns) {DropHandshakes:469(3170ns) {DropIncomingHandshake:441(380ns) DropOutgoingHandshake:456(383ns)} GenerateSessionVirtualId:988(785ns) PrepareNewSessionHandshake:476(5268ns) {SwitchToState:178(525ns)}}}}}
2025-11-11T17:18:44.521837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518496297220557:2074] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:44.707393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518495707878180:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(304654010ns) {Configure:104(304647296ns) {ProcessConfigured:135(56979ns) {StartInitialHandshake:157(54805ns) {DropHandshakes:469(2452ns) {DropIncomingHandshake:441(400ns) DropOutgoingHandshake:456(352ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:476(686ns) {SwitchToState:178(326ns)}}}}}
2025-11-11T17:18:44.718220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518495707878152:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.602066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188017:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.745881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:44.753842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518495707878152:2069] [node 5] ICP22 created new session: [3:7571518495707878205:2069]
2025-11-11T17:18:44.755324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518495707878206:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48670
2025-11-11T17:18:44.773668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518495707878205:2069] [node 5] ICS09 handshake done sender: [3:7571518495707878191:2115] self: [3:7571518495707878190:0] peer: [5:7571518496297220622:0] socket: 174 qp: -1
2025-11-11T17:18:44.774588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518495707878205:2069] [node 5] ICS10 traffic start
2025-11-11T17:18:44.784426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:44.787213Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878194:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.787245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518495707878194:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.789871Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878206:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.810021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518491412910822:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(516079541ns) {Configure:104(516072137ns) {ProcessConfigured:135(27545627ns) {StartInitialHandshake:157(27542714ns) {DropHandshakes:469(1990ns) {DropIncomingHandshake:441(508ns) DropOutgoingHandshake:456(377ns)} GenerateSessionVirtualId:988(902ns) PrepareNewSessionHandshake:476(3475ns) {SwitchToState:178(1130ns)}}}}}
2025-11-11T17:18:44.818342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518495707878209:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48670
2025-11-11T17:18:44.821669Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878202:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.821709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518495707878202:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.830965Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518495707878202:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:44.831213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518495707878180:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.831284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:44.831328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518495707878180:2070] [node 6] ICP22 created new session: [3:7571518495707878211:2070]
2025-11-11T17:18:44.831386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518495707878211:2070] [node 6] ICS09 handshake done sender: [3:7571518495707878202:2118] self: [3:7571518495707878201:0] peer: [6:7571518494091188029:0] socket: 153 qp: -1
2025-11-11T17:18:44.831420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518495707878211:2070] [node 6] ICS10 traffic start
2025-11-11T17:18:44.831471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:44.854220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518495707878152:2069] [node 5] ICP35 event processing took too much time PendingConnection:278(135800660ns) {HandleHandshakeStatus:329(135797318ns) {DropIncomingHandshake:441(22062ns) DropOutgoingHandshake:456(499ns) SwitchToState:178(407ns) ProcessPendingSessionEvents:495(12321420ns) {ForwardSessionEventToSession:612(12272994ns) ForwardSessionEventToSession:612(5584ns) ForwardSessionEventToSession:612(7871ns) ForwardSessionEventToSession:612(1951ns) ForwardSessionEventToSession:612(4477ns) ForwardSessionEventToSession:612(2578ns) ForwardSessionEventToSession:612(3111ns) ForwardSessionEventToSession:612(3814ns) ForwardSessionEventToSession:612(4788ns) ForwardSessionEventToSession:612(2791ns)}}}
2025-11-11T17:18:44.538205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220609:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:44.881755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:44.889650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518492918304146:2096] [node 6] ICP35 event processing took too much time PendingConnection:278(19873741ns) {IncomingHandshake:261(19872143ns) {DropIncomingHandshake:441(206ns) PrepareNewSessionHandshake:476(1330ns) {SwitchToState:178(476ns)} GenerateSessionVirtualId:988(443ns) IssueIncomingHandshakeReply:193(6820ns)}}
2025-11-11T17:18:44.598142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518491707265379:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(460121422ns) {Configure:104(435891341ns) {ProcessConfigured:135(7909742ns) {StartInitialHandshake:157(7907568ns) {DropHandshakes:469(623ns) {DropIncomingHandshake:441(243ns) DropOutgoingHandshake:456(205ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(2664ns) {SwitchToState:178(914ns)}}}}}
2025-11-11T17:18:44.629951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518488108330777:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(415899848ns) {Configure:104(415895991ns) {ProcessConfigured:135(27758978ns) {StartInitialHandshake:157(12133936ns) {DropHandshakes:469(2985ns) {DropIncomingHandshake:441(607ns) DropOutgoingHandshake:456(340ns)} GenerateSessionVirtualId:988(2910ns) PrepareNewSessionHandshake:476(2721ns) {SwitchToState:178(959ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.688136Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232743:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.650666Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518494091187966:2070] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:44.678602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298156:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.665632Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24835) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:44.681835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518494091187952:2062] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24835) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518494091187966:2070] held: no
2025-11-11T17:18:44.682380Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518494091187967:2071] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:44.689722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518494091187952:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.689786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:44.689826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518494091187952:2062] [node 5] ICP22 created new session: [6:7571518494091188018:2062]
2025-11-11T17:18:44.710420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188019:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.718458Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518494091188019:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.718492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518494091187947:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.737529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:44.737549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518494091188018:2062] [node 5] ICS09 handshake done sender: [6:7571518494091187966:2070] self: [6:7571518494091188014:0] peer: [5:7571518496297220556:0] socket: 161 qp: -1
2025-11-11T17:18:44.747042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518494091187947:2059] [node 4] ICP22 created new session: [6:7571518494091188020:2059]
2025-11-11T17:18:44.747100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518494091188018:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:44.747144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:44.747409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518489796220638:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.756418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188025:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.756469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188026:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.756570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188027:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.762912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518494091187952:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(73062859ns) {HandleHandshakeStatus:329(73058190ns) {DropIncomingHandshake:441(344ns) DropOutgoingHandshake:456(204ns) SwitchToState:178(279ns) ProcessPendingSessionEvents:495(1220ns)}}
2025-11-11T17:18:44.773877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518494091188020:2059] [node 4] ICS09 handshake done sender: [6:7571518494091187967:2071] self: [6:7571518494091188012:0] peer: [4:7571518492403298129:0] socket: 165 qp: -1
2025-11-11T17:18:44.773923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518494091188020:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:44.774004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:44.789267Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518494091188019:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:44.801981Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518494091188026:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.807886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518489796220640:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.814721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518489796220638:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.821865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518494091188025:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.822315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518494091187947:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(103620779ns) {HandleHandshakeStatus:329(103619109ns) {DropIncomingHandshake:441(854ns) DropOutgoingHandshake:456(4750ns) SwitchToState:178(310ns) ProcessPendingSessionEvents:495(122458ns) {ForwardSessionEventToSession:612(58557ns) ForwardSessionEventToSession:612(2899ns) ForwardSessionEventToSession:612(2946ns) ForwardSessionEventToSession:612(10472ns) ForwardSessionEventToSession:612(8615ns) ForwardSessionEventToSession:612(16209ns) ForwardSessionEventToSession:612(2096ns) ForwardSessionEventToSession:612(2667ns) ForwardSessionEventToSession:612(4959ns) ForwardSessionEventToSession:612(2885ns)}}}
2025-11-11T17:18:44.841800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518489796220639:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.841935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518489796220639:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(18814330ns) {IncomingHandshake:261(18812806ns) {DropIncomingHandshake:441(470ns) PrepareNewSessionHandshake:476(1356ns) {SwitchToState:178(307ns)} GenerateSessionVirtualId:988(594ns) IssueIncomingHandshakeReply:193(46053ns) {SwitchToState:178(497ns)}}}
2025-11-11T17:18:44.861677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:44.869691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518489796220638:2048] [node 1] ICP22 created new session: [6:7571518494091188031:2048]
2025-11-11T17:18:44.909846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518494091188031:2048] [node 1] ICS09 handshake done sender: [6:7571518494091188019:2119] self: [6:7571518494091188022:0] peer: [1:7571518496002232749:0] socket: 178 qp: -1
2025-11-11T17:18:44.914295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518494091188031:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:44.919425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188033:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.919633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518494091188032:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15175
2025-11-11T17:18:44.921668Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7870) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:44.925810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:44.926259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518489796220638:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(111340967ns) {HandleHandshakeStatus:329(111338466ns) {DropIncomingHandshake:441(2158ns) DropOutgoingHandshake:456(10529ns) SwitchToState:178(831ns) ProcessPendingSessionEvents:495(90690ns) {ForwardSessionEventToSession:612(49299ns) ForwardSessionEventToSession:612(2845ns) ForwardSessionEventToSession:612(8042ns) ForwardSessionEventToSession:612(3094ns) ForwardSessionEventToSession:612(4643ns) ForwardSessionEventToSession:612(4898ns) ForwardSessionEventToSession:612(2409ns) ForwardSessionEventToSession:612(3947ns) ForwardSessionEventToSession:612(2352ns) ForwardSessionEventToSession:612(1969ns)}}}
2025-11-11T17:18:44.893054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518495707878194:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:44.941748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518489796220640:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7870) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518494091188026:2121] held: no
2025-11-11T17:18:44.943814Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518494091188025:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:44.943983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518489796220639:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.944029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:44.944077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518489796220639:2049] [node 2] ICP22 created new session: [6:7571518494091188035:2049]
2025-11-11T17:18:44.944133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518494091188035:2049] [node 2] ICS09 handshake done sender: [6:7571518494091188025:2120] self: [6:7571518494091188030:0] peer: [2:7571518492918304161:0] socket: 154 qp: -1
2025-11-11T17:18:44.944163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518494091188035:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:44.944198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:44.688157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518496002232743:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.891730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518492918304128:2075] [node 4] ICP22 created new session: [2:7571518492918304173:2075]
2025-11-11T17:18:44.538380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220610:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:44.917940Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878209:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.620503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518496297220555:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.678748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298157:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.893181Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304171:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.678881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298158:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.688869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518496002232759:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14578
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.918907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518495707878216:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48670
2025-11-11T17:18:44.893625Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304163:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.893659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518492918304163:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.909918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518492918304173:2075] [node 4] ICS09 handshake done sender: [2:7571518492918304158:2118] self: [2:7571518492918304157:0] peer: [4:7571518492403298162:0] socket: 190 qp: -1
2025-11-11T17:18:44.909955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518492918304173:2075] [node 4] ICS10 traffic start
2025-11-11T17:18:44.921700Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:44.922098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:44.929776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518492918304163:2120] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:44.941835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518488623336779:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518492918304166:2122] held: no
2025-11-11T17:18:44.981833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518492918304146:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.990071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:44.994381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518492918304146:2096] [node 6] ICP22 created new session: [2:7571518492918304179:2096]
2025-11-11T17:18:45.009703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518492918304179:2096] [node 6] ICS09 handshake done sender: [2:7571518492918304163:2120] self: [2:7571518492918304161:0] peer: [6:7571518494091188030:0] socket: 152 qp: -1
2025-11-11T17:18:45.009758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518492918304179:2096] [node 6] ICS10 traffic start
2025-11-11T17:18:45.009818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:45.010354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518492918304146:2096] [node 6] ICP35 event processing took too much time PendingConnection:278(28298853ns) {HandleHandshakeStatus:329(28295558ns) {DropIncomingHandshake:441(13947ns) DropOutgoingHandshake:456(437ns) SwitchToState:178(462ns) ProcessPendingSessionEvents:495(100133ns) {ForwardSessionEventToSession:612(57044ns) ForwardSessionEventToSession:612(5400ns) ForwardSessionEventToSession:612(2388ns) ForwardSessionEventToSession:612(2247ns) ForwardSessionEventToSession:612(7172ns) ForwardSessionEventToSession:612(1944ns) ForwardSessionEventToSession:612(3174ns) ForwardSessionEventToSession:612(5810ns) ForwardSessionEventToSession:612(2720ns) ForwardSessionEventToSession:612(4369ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:44.620589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:44.620639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518496297220555:2070] [node 6] ICP22 created new session: [5:7571518496297220613:2070]
2025-11-11T17:18:44.620690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518496297220613:2070] [node 6] ICS09 handshake done sender: [5:7571518496297220557:2074] self: [5:7571518496297220556:0] peer: [6:7571518494091188014:0] socket: 160 qp: -1
2025-11-11T17:18:44.620725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518496297220613:2070] [node 6] ICS10 traffic start
2025-11-11T17:18:44.620766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:44.621583Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518496297220547:2066] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:44.621903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518492002253233:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(431915766ns) {Configure:104(431911843ns) {ProcessConfigured:135(19882589ns) {StartInitialHandshake:157(19878240ns) {DropHandshakes:469(2195ns) {DropIncomingHandshake:441(277ns) DropOutgoingHandshake:456(446ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:476(2050ns) {SwitchToState:178(853ns)}}}}}
2025-11-11T17:18:44.641978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518496297220546:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.642030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:44.642076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518496297220546:2061] [node 4] ICP22 created new session: [5:7571518496297220619:2061]
2025-11-11T17:18:44.642117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518496297220619:2061] [node 4] ICS09 handshake done sender: [5:7571518496297220547:2066] self: [5:7571518496297220602:0] peer: [4:7571518492403298103:0] socket: 151 qp: -1
2025-11-11T17:18:44.642146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518496297220619:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:44.642178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:44.651249Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518496297220609:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.655672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220618:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:44.663550Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518496297220598:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.663668Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:44.673899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518492002253231:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(495899961ns) {Configure:104(495897164ns) {ProcessConfigured:135(34041015ns) {StartInitialHandshake:157(34038316ns) {DropHandshakes:469(2011ns) {DropIncomingHandshake:441(353ns) DropOutgoingHandshake:456(428ns)} GenerateSessionVirtualId:988(1037ns) PrepareNewSessionHandshake:476(1458ns) {SwitchToState:178(134ns)}}}}}
2025-11-11T17:18:44.674031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518492002253232:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518496297220554:2073] held: no
2025-11-11T17:18:44.674209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518492002253233:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.686459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518492002253231:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.686703Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518496297220554:2073] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:44.686904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518492002253232:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.686955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:44.686999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518492002253232:2049] [node 2] ICP22 created new session: [5:7571518496297220624:2049]
2025-11-11T17:18:44.687043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518496297220624:2049] [node 2] ICS09 handshake done sender: [5:7571518496297220554:2073] self: [5:7571518496297220606:0] peer: [2:7571518492918304149:0] socket: 157 qp: -1
2025-11-11T17:18:44.687072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518496297220624:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:44.687099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:44.710079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220626:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:44.710604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518496297220627:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:865
2025-11-11T17:18:44.718655Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518496297220609:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:44.719233Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518496297220598:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:44.719430Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7870) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:44.720210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518492002253233:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.720278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:44.720322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518492002253233:2050] [node 3] ICP22 created new session: [5:7571518496297220628:2050]
2025-11-11T17:18:44.720357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518496297220628:2050] [node 3] ICS09 handshake done sender: [5:7571518496297220609:2117] self: [5:7571518496297220622:0] peer: [3:7571518495707878190:0] socket: 175 qp: -1
2025-11-11T17:18:44.720388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518496297220628:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:44.720438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:44.720660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518492002253233:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:44.720806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518492002253231:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.720868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:44.720900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518492002253231:2048] [node 1] ICP22 created new session: [5:7571518496297220630:2048]
2025-11-11T17:18:44.720930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518496297220630:2048] [node 1] ICS09 handshake done sender: [5:7571518496297220598:2113] self: [5:7571518496297220623:0] peer: [1:7571518496002232742:0] socket: 171 qp: -1
2025-11-11T17:18:44.720959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518496297220630:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:44.720982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:45.014970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518492918304128:2075] [node 4] ICP35 event processing took too much time PendingConnection:278(163589749ns) {HandleHandshakeStatus:329(163586670ns) {DropIncomingHandshake:441(14068ns) DropOutgoingHandshake:456(332ns) SwitchToState:178(2948ns) ProcessPendingSessionEvents:495(24699961ns) {ForwardSessionEventToSession:612(24648126ns) ForwardSessionEventToSession:612(5492ns) ForwardSessionEventToSession:612(9186ns) ForwardSessionEventToSession:612(2439ns) ForwardSessionEventToSession:612(2895ns) ForwardSessionEventToSession:612(4817ns) ForwardSessionEventToSession:612(2517ns) ForwardSessionEventToSession:612(5355ns) ForwardSessionEventToSession:612(5239ns) ForwardSessionEventToSession:612(2886ns)}}}
2025-11-11T17:18:44.685828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518492403298102:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.690342Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518496002232743:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:44.919839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518491412910821:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.958989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518495707878184:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.959069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:44.959114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518495707878184:2068] [node 4] ICP22 created new session: [3:7571518495707878217:2068]
2025-11-11T17:18:44.959152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518495707878217:2068] [node 4] ICS09 handshake done sender: [3:7571518495707878194:2116] self: [3:7571518495707878193:0] peer: [4:7571518492403298163:0] socket: 186 qp: -1
2025-11-11T17:18:44.959184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518495707878217:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:44.959218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:45.003354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518500002845515:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48670
2025-11-11T17:18:44.690716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518496002232736:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.685879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518492403298128:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:45.030366Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518495707878216:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:45.033821Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518495707878209:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:45.037835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518491412910822:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:45.037901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518491412910821:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:45.037984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:45.038025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518491412910821:2048] [node 1] ICP22 created new session: [3:7571518500002845517:2048]
2025-11-11T17:18:45.038082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518500002845517:2048] [node 1] ICS09 handshake done sender: [3:7571518495707878209:2121] self: [3:7571518495707878215:0] peer: [1:7571518496002232754:0] socket: 211 qp: -1
2025-11-11T17:18:45.038116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518500002845517:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:45.038161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:45.048767Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:45.057722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518491412910821:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:45.015618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518492918304177:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-11-11T17:18:44.705886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518492403298158:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.724280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:44.733917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518492403298102:2063] [node 5] ICP22 created new session: [4:7571518492403298160:2063]
2025-11-11T17:18:44.733975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518492403298160:2063] [node 5] ICS09 handshake done sender: [4:7571518492403298104:2077] self: [4:7571518492403298103:0] peer: [5:7571518496297220602:0] socket: 150 qp: -1
2025-11-11T17:18:44.734017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518492403298160:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:44.734067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:44.734604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518492403298102:2063] [node 5] ICP35 event processing took too much time PendingConnection:278(64544017ns) {HandleHandshakeStatus:329(48524063ns) {DropIncomingHandshake:441(15557ns) DropOutgoingHandshake:456(243ns) SwitchToState:178(764ns) ProcessPendingSessionEvents:495(102453ns) {ForwardSessionEventToSession:612(54751ns) ForwardSessionEventToSession:612(6268ns) ForwardSessionEventToSession:612(2631ns) ForwardSessionEventToSession:612(5107ns) ForwardSessionEventToSession:612(3365ns) ForwardSessionEventToSession:612(2020ns) ForwardSessionEventToSession:612(8495ns) ForwardSessionEventToSession:612(2921ns) ForwardSessionEventToSession:612(2495ns) ForwardSessionEventToSession:612(5536ns)}}}
2025-11-11T17:18:44.734745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518492403298157:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.734808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518492403298156:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.735478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518488108330777:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.735534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518488108330778:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.735578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518488108330776:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:44.737783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:44.773715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518492403298128:2092] [node 6] ICP22 created new session: [4:7571518492403298165:2092]
2025-11-11T17:18:44.805757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518492403298165:2092] [node 6] ICS09 handshake done sender: [4:7571518492403298130:2100] self: [4:7571518492403298129:0] peer: [6:7571518494091188012:0] socket: 162 qp: -1
2025-11-11T17:18:44.808424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298167:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.808921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298168:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.814119Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518492403298158:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:44.817751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518492403298165:2092] [node 6] ICS10 traffic start
2025-11-11T17:18:44.821844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:44.822893Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518492403298156:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:44.837776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518488108330777:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.845947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518488108330776:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.845997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:44.846042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518488108330776:2048] [node 1] ICP22 created new session: [4:7571518492403298170:2048]
2025-11-11T17:18:44.846076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518492403298170:2048] [node 1] ICS09 handshake done sender: [4:7571518492403298156:2117] self: [4:7571518492403298164:0] peer: [1:7571518496002232746:0] socket: 172 qp: -1
2025-11-11T17:18:44.846107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518492403298170:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:44.846165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:44.857700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:44.861804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518488108330777:2049] [node 2] ICP22 created new session: [4:7571518492403298172:2049]
2025-11-11T17:18:44.890301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518492403298128:2092] [node 6] ICP35 event processing took too much time PendingConnection:278(225294262ns) {HandleHandshakeStatus:329(204176814ns) {DropIncomingHandshake:441(11878ns) DropOutgoingHandshake:456(290ns) SwitchToState:178(419ns) ProcessPendingSessionEvents:495(22938706ns) {ForwardSessionEventToSession:612(22890408ns) ForwardSessionEventToSession:612(5706ns) ForwardSessionEventToSession:612(6235ns) ForwardSessionEventToSession:612(2656ns) ForwardSessionEventToSession:612(6584ns) ForwardSessionEventToSession:612(2766ns) ForwardSessionEventToSession:612(2406ns) ForwardSessionEventToSession:612(4249ns) ForwardSessionEventToSession:612(4836ns) ForwardSessionEventToSession:612(2373ns)}}}
2025-11-11T17:18:44.891634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518492403298172:2049] [node 2] ICS09 handshake done sender: [4:7571518492403298158:2119] self: [4:7571518492403298162:0] peer: [2:7571518492918304157:0] socket: 192 qp: -1
2025-11-11T17:18:44.897774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518492403298172:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:44.897836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:44.919127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518492403298174:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-11T17:18:44.969911Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518492403298157:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:44.970292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518488108330778:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:44.970348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:44.970394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518488108330778:2050] [node 3] ICP22 created new session: [4:7571518492403298175:2050]
2025-11-11T17:18:44.970442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518492403298175:2050] [node 3] ICS09 handshake done sender: [4:7571518492403298157:2118] self: [4:7571518492403298163:0] peer: [3:7571518495707878193:0] socket: 191 qp: -1
2025-11-11T17:18:44.970474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518492403298175:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:44.970513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:44.978383Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518488108330777:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(140400820ns) {HandleHandshakeStatus:329(140398257ns) {DropIncomingHandshake:441(717ns) DropOutgoingHandshake:456(27405ns) SwitchToState:178(523ns) ProcessPendingSessionEvents:495(12272754ns) {ForwardSessionEventToSession:612(12224290ns) ForwardSessionEventToSession:612(6773ns) ForwardSessionEventToSession:612(7789ns) ForwardSessionEventToSession:612(2453ns) ForwardSessionEventToSession:612(2351ns) ForwardSessionEventToSession:612(3614ns) ForwardSessionEventToSession:612(2111ns) ForwardSessionEventToSession:612(5575ns) ForwardSessionEventToSession:612(2555ns) ForwardSessionEventToSession:612(5370ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:45.019336Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518492918304166:2122] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:45.029678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518488623336779:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:45.053395Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518492918304164:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:45.053424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518492918304164:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:45.065627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:45.071283Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518492918304164:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:45.074434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518488623336780:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:45.074496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:45.074533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518488623336780:2049] [node 3] ICP22 created new session: [2:7571518497213271479:2049]
2025-11-11T17:18:45.074577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518497213271479:2049] [node 3] ICS09 handshake done sender: [2:7571518492918304164:2121] self: [2:7571518492918304160:0] peer: [3:7571518500002845516:0] socket: 159 qp: -1
2025-11-11T17:18:45.074608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518497213271479:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:45.074632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:45.069976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518500002845519:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48670
2025-11-11T17:18:44.691358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:44.705944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518496002232736:2094] [node 5] ICP22 created new session: [1:7571518496002232761:2094]
2025-11-11T17:18:44.742766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518496002232761:2094] [node 5] ICS09 handshake done sender: [1:7571518496002232743:2112] self: [1:7571518496002232742:0] peer: [5:7571518496297220623:0] socket: 170 qp: -1
2025-11-11T17:18:44.748273Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232750:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.748293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518496002232750:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.749184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518496002232763:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14578
2025-11-11T17:18:44.757780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518496002232761:2094] [node 5] ICS10 traffic start
2025-11-11T17:18:44.757997Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:44.765809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:44.766729Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518496002232750:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:44.769629Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232747:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.769669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518496002232747:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.790024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518496002232738:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.805776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518496002232747:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:44.805889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:44.817685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518496002232738:2095] [node 6] ICP22 created new session: [1:7571518496002232767:2095]
2025-11-11T17:18:44.821817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518496002232767:2095] [node 6] ICS09 handshake done sender: [1:7571518496002232750:2115] self: [1:7571518496002232749:0] peer: [6:7571518494091188022:0] socket: 185 qp: -1
2025-11-11T17:18:44.826463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518496002232736:2094] [node 5] ICP35 event processing took too much time PendingConnection:278(135551115ns) {HandleHandshakeStatus:329(135544215ns) {DropIncomingHandshake:441(311ns) DropOutgoingHandshake:456(368ns) SwitchToState:178(506ns) ProcessPendingSessionEvents:495(8495230ns) {ForwardSessionEventToSession:612(8442472ns) ForwardSessionEventToSession:612(5499ns) ForwardSessionEventToSession:612(3351ns) ForwardSessionEventToSession:612(8906ns) ForwardSessionEventToSession:612(2269ns) ForwardSessionEventToSession:612(3275ns) ForwardSessionEventToSession:612(7261ns) ForwardSessionEventToSession:612(2241ns) ForwardSessionEventToSession:612(4546ns) ForwardSessionEventToSession:612(2444ns)}}}
2025-11-11T17:18:44.837657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518496002232767:2095] [node 6] ICS10 traffic start
2025-11-11T17:18:44.846802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:44.849195Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232755:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.849235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518496002232755:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.865808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518496002232739:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.865850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:44.865905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518496002232739:2096] [node 4] ICP22 created new session: [1:7571518496002232770:2096]
2025-11-11T17:18:44.865949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518496002232770:2096] [node 4] ICS09 handshake done sender: [1:7571518496002232747:2114] self: [1:7571518496002232746:0] peer: [4:7571518492403298164:0] socket: 183 qp: -1
2025-11-11T17:18:44.865984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518496002232770:2096] [node 4] ICS10 traffic start
2025-11-11T17:18:44.866016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:44.883987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518496002232755:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:44.884363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518491707265378:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:44.884440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:44.884482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518491707265378:2048] [node 2] ICP22 created new session: [1:7571518496002232773:2048]
2025-11-11T17:18:44.884524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518496002232773:2048] [node 2] ICS09 handshake done sender: [1:7571518496002232755:2116] self: [1:7571518496002232753:0] peer: [2:7571518492918304170:0] socket: 158 qp: -1
2025-11-11T17:18:44.884554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518496002232773:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:44.884605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:44.885819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232763:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.893811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518496002232739:2096] [node 4] ICP18 handshake (actor [1:7571518496002232763:2119]) is too old
2025-11-11T17:18:44.954200Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518496002232757:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:44.954246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518496002232757:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:44.989848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518496002232757:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:45.017415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518491707265379:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:45.025951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:45.041802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518491707265379:2049] [node 3] ICP22 created new session: [1:7571518500297200072:2049]
2025-11-11T17:18:45.053235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518496002232738:2095] [node 6] ICP35 event processing took too much time PendingConnection:278(195673019ns) {HandleHandshakeStatus:329(195670695ns) {DropIncomingHandshake:441(12796ns) DropOutgoingHandshake:456(623ns) SwitchToState:178(366ns) ProcessPendingSessionEvents:495(99940680ns) {ForwardSessionEventToSession:612(46836144ns) ForwardSessionEventToSession:612(4871ns) ForwardSessionEventToSession:612(2315ns) ForwardSessionEventToSession:612(14630359ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(15265192ns) ForwardSessionEventToSession:612(8083703ns) ForwardSessionEventToSession:612(7965ns) ForwardSessionEventToSession:612(7925186ns) ForwardSessionEventToSession:612(3985ns)}}}
2025-11-11T17:18:45.064124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518500297200072:2049] [node 3] ICS09 handshake done sender: [1:7571518496002232757:2117] self: [1:7571518496002232754:0] peer: [3:7571518495707878215:0] socket: 210 qp: -1
2025-11-11T17:18:45.064160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518500297200072:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:45.064201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:45.078214Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518491707265379:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(60610529ns) {HandleHandshakeStatus:329(60608203ns) {DropIncomingHandshake:441(44791ns) DropOutgoingHandshake:456(497ns) SwitchToState:178(368ns) ProcessPendingSessionEvents:495(6823210ns) {ForwardSessionEventToSession:612(6774524ns) ForwardSessionEventToSession:612(7947ns) ForwardSessionEventToSession:612(4818ns) ForwardSessionEventToSession:612(2326ns) ForwardSessionEventToSession:612(2527ns) ForwardSessionEventToSession:612(4823ns) ForwardSessionEventToSession:612(3013ns) ForwardSessionEventToSession:612(5244ns) ForwardSessionEventToSession:612(3348ns) ForwardSessionEventToSession:612(4996ns)}}}
2025-11-11T17:18:45.041691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518489796220640:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(52058729ns) {HandleHandshakeStatus:395(48052432ns) {LogHandshakeFail:472(12070010ns) {UpdateErrorStateLog:544(9003ns)} DropOutgoingHandshake:456(461ns) ProcessPendingSessionEvents:495(35909503ns) {DropSessionEvent:512(14591ns) DropSessionEvent:512(8099531ns) DropSessionEvent:512(13358ns) DropSessionEvent:512(3827ns) DropSessionEvent:512(3934ns) DropSessionEvent:512(11970506ns) DropSessionEvent:512(19033ns) DropSessionEvent:512(8641ns) DropSessionEvent:512(3231ns) DropSessionEvent:512(26023ns)}}}
2025-11-11T17:18:45.053948Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518494091188026:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:45.063233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518489796220640:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:45.063292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:45.063340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518489796220640:2050] [node 3] ICP22 created new session: [6:7571518498386155333:2050]
2025-11-11T17:18:45.063380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518498386155333:2050] [node 3] ICS09 handshake done sender: [6:7571518494091188026:2121] self: [6:7571518494091188029:0] peer: [3:7571518495707878201:0] socket: 204 qp: -1
2025-11-11T17:18:45.063411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518498386155333:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:45.063450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:45.089985Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:45.078790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518488623336779:2048] [node 1] ICP22 created new session: [2:7571518497213271478:2048]
2025-11-11T17:18:45.093376Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518495707878216:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:45.097802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518491412910822:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518495707878216:2122] held: no
2025-11-11T17:18:45.078857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518497213271478:2048] [node 1] ICS09 handshake done sender: [2:7571518492918304166:2122] self: [2:7571518492918304170:0] peer: [1:7571518496002232753:0] socket: 200 qp: -1
2025-11-11T17:18:45.078902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518497213271478:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:45.078966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:45.079388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518488623336779:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(57441571ns) {HandleHandshakeStatus:329(57439752ns) {DropIncomingHandshake:441(2679ns) DropOutgoingHandshake:456(329ns) SwitchToState:178(609ns) ProcessPendingSessionEvents:495(912ns)}}
2025-11-11T17:18:45.121880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518491412910822:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:45.121970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:45.122026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518491412910822:2049] [node 2] ICP22 created new session: [3:7571518500002845520:2049]
2025-11-11T17:18:45.122071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518500002845520:2049] [node 2] ICS09 handshake done sender: [3:7571518495707878216:2122] self: [3:7571518500002845516:0] peer: [2:7571518492918304160:0] socket: 194 qp: -1
2025-11-11T17:18:45.122105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518500002845520:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:45.122139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:45.209555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:45.209804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518497213271478:2048] [node 1] ICS01 socket: 200 reason# EndOfStream
2025-11-11T17:18:45.209843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518488623336779:2048] [node 1] ICP30 unregister session Session# [2:7571518497213271478:2048] VirtualId# [2:7571518492918304170:0]
2025-11-11T17:18:45.209875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518497213271478:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.209946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:45.209981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:45.212269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:45.212479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518500002845517:2048] [node 1] ICS01 socket: 211 reason# EndOfStream
2025-11-11T17:18:45.212516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518491412910821:2048] [node 1] ICP30 unregister session Session# [3:7571518500002845517:2048] VirtualId# [3:7571518495707878215:0]
2025-11-11T17:18:45.212546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518500002845517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.212607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:45.212641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:45.225740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:45.226017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518496297220630:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-11T17:18:45.226070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518492002253231:2048] [node 1] ICP30 unregister session Session# [5:7571518496297220630:2048] VirtualId# [5:7571518496297220623:0]
2025-11-11T17:18:45.226105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518496297220630:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.226204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:45.226277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:45.229728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:45.230027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518494091188031:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-11T17:18:45.230107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518489796220638:2048] [node 1] ICP30 unregister session Session# [6:7571518494091188031:2048] VirtualId# [6:7571518494091188022:0]
2025-11-11T17:18:45.230145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518494091188031:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.230244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:45.230306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:45.229717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:45.230042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518492403298170:2048] [node 1] ICS01 socket: 172 reason# EndOfStream
2025-11-11T17:18:45.230096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518488108330776:2048] [node 1] ICP30 unregister session Session# [4:7571518492403298170:2048] VirtualId# [4:7571518492403298164:0]
2025-11-11T17:18:45.230139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518492403298170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.230236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:45.230291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:45.305813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:45.306038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518500002845520:2049] [node 2] ICS01 socket: 194 reason# EndOfStream
2025-11-11T17:18:45.306076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518491412910822:2049] [node 2] ICP30 unregister session Session# [3:7571518500002845520:2049] VirtualId# [3:7571518500002845516:0]
2025-11-11T17:18:45.306107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518500002845520:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.306201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:45.306242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:45.317728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:45.317727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:45.317759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:45.317944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518492403298172:2049] [node 2] ICS01 socket: 192 reason# EndOfStream
2025-11-11T17:18:45.317943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518496297220624:2049] [node 2] ICS01 socket: 157 reason# EndOfStream
2025-11-11T17:18:45.318016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518494091188035:2049] [node 2] ICS01 socket: 154 reason# EndOfStream
2025-11-11T17:18:45.318046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518489796220639:2049] [node 2] ICP30 unregister session Session# [6:7571518494091188035:2049] VirtualId# [6:7571518494091188030:0]
2025-11-11T17:18:45.317979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518488108330777:2049] [node 2] ICP30 unregister session Session# [4:7571518492403298172:2049] VirtualId# [4:7571518492403298162:0]
2025-11-11T17:18:45.318071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518494091188035:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.318008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518492403298172:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.317979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518492002253232:2049] [node 2] ICP30 unregister session Session# [5:7571518496297220624:2049] VirtualId# [5:7571518496297220606:0]
2025-11-11T17:18:45.318132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:45.318080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:45.318009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518496297220624:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.318121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:45.318097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:45.318143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:45.318168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:45.381733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:45.381733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:45.381733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:45.381940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518492403298175:2050] [node 3] ICS01 socket: 191 reason# EndOfStream
2025-11-11T17:18:45.381930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518498386155333:2050] [node 3] ICS01 socket: 204 reason# EndOfStream
2025-11-11T17:18:45.381973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518488108330778:2050] [node 3] ICP30 unregister session Session# [4:7571518492403298175:2050] VirtualId# [4:7571518492403298163:0]
2025-11-11T17:18:45.381997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518496297220628:2050] [node 3] ICS01 socket: 175 reason# EndOfStream
2025-11-11T17:18:45.381967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518489796220640:2050] [node 3] ICP30 unregister session Session# [6:7571518498386155333:2050] VirtualId# [6:7571518494091188029:0]
2025-11-11T17:18:45.381999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518492403298175:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.382088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:45.382023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518498386155333:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.382128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:45.382099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:45.382133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:45.382043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518492002253233:2050] [node 3] ICP30 unregister session Session# [5:7571518496297220628:2050] VirtualId# [5:7571518496297220622:0]
2025-11-11T17:18:45.382066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518496297220628:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.382125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:45.382160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:45.533776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:45.533993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518496297220619:2061] [node 4] ICS01 socket: 151 reason# EndOfStream
2025-11-11T17:18:45.534032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518496297220546:2061] [node 4] ICP30 unregister session Session# [5:7571518496297220619:2061] VirtualId# [5:7571518496297220602:0]
2025-11-11T17:18:45.534061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518496297220619:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.534145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:45.534195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:45.534293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:45.534467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518494091188020:2059] [node 4] ICS01 socket: 165 reason# EndOfStream
2025-11-11T17:18:45.534495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518494091187947:2059] [node 4] ICP30 unregister session Session# [6:7571518494091188020:2059] VirtualId# [6:7571518494091188012:0]
2025-11-11T17:18:45.534516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518494091188020:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.534571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:45.534607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:45.589726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:45.589936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518494091188018:2062] [node 5] ICS01 socket: 161 reason# EndOfStream
2025-11-11T17:18:45.589978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518494091187952:2062] [node 5] ICP30 unregister session Session# [6:7571518494091188018:2062] VirtualId# [6:7571518494091188014:0]
2025-11-11T17:18:45.590013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518494091188018:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:45.590128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:45.590169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:45.737965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518499826895761:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:45.738008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518499826895762:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:45.771426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518497998670214:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:45.771493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518497998670215:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:45.822356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518498253667188:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:45.822578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518498253667189:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:45.861939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518497516911587:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:45.861988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518497516911588:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:45.862012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518497516911589:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:45.941924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518496691278106:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:45.941973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518496691278107:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:45.941996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518496691278108:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:46.013159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518498198262032:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:46.013201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518498198262033:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:46.013228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518498198262034:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:46.113237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518498198262046:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:46.214378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518501811878901:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:46.211585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518496691278108:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(99320241ns) {Configure:104(99315811ns) {ProcessConfigured:135(61647569ns) {StartInitialHandshake:157(61643852ns) {DropHandshakes:469(2585ns) {DropIncomingHandshake:441(503ns) DropOutgoingHandshake:456(450ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:476(8726ns) {SwitchToState:178(3425ns)}}}}}
2025-11-11T17:18:46.296198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518504121863120:2074] [node 5] ICP01 ready to work
2025-11-11T17:18:46.222712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518500986245425:2066] [node 4] ICP01 ready to work
2025-11-11T17:18:46.326548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518501811878944:2063] [node 6] ICP01 ready to work
2025-11-11T17:18:46.354434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518504121863121:2075] [node 6] ICP01 ready to work
2025-11-11T17:18:46.356027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518502548634547:2061] [node 5] ICP01 ready to work
2025-11-11T17:18:46.429969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518502548634548:2063] [node 6] ICP01 ready to work
2025-11-11T17:18:46.430284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518502548634549:2069] [node 4] ICP01 ready to work
2025-11-11T17:18:46.429855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518504121863122:2076] [node 4] ICP01 ready to work
2025-11-11T17:18:46.353967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518502293637571:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:46.250721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518496691278106:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25176805ns) {Configure:104(25173050ns) {ProcessConfigured:135(65542ns) {StartInitialHandshake:157(62803ns) {DropHandshakes:469(2527ns) {DropIncomingHandshake:441(948ns) DropOutgoingHandshake:456(346ns)} GenerateSessionVirtualId:988(653ns) PrepareNewSessionHandshake:476(1369ns) {SwitchToState:178(487ns)}}}}}
2025-11-11T17:18:46.446109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518498198262046:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(332545888ns) {Configure:104(332543125ns) {ProcessConfigured:135(83377ns) {StartInitialHandshake:157(79185ns) {DropHandshakes:469(1539ns) {DropIncomingHandshake:441(398ns) DropOutgoingHandshake:456(178ns)} GenerateSessionVirtualId:988(487ns) PrepareNewSessionHandshake:476(1869ns) {SwitchToState:178(525ns)}}}}}
2025-11-11T17:18:46.255026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518500986245441:2077] [node 6] ICP01 ready to work
2025-11-11T17:18:46.461834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518502293637574:2065] [node 4] ICP01 ready to work
2025-11-11T17:18:46.466093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518502493229393:2080] [node 5] ICP01 ready to work
2025-11-11T17:18:46.480381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518502548634547:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19466037ns) {Configure:104(19460993ns) {ProcessConfigured:135(51864ns) {StartInitialHandshake:157(49537ns) {DropHandshakes:469(1791ns) {DropIncomingHandshake:441(321ns) DropOutgoingHandshake:456(333ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:476(1341ns) {SwitchToState:178(465ns)}}}}}
2025-11-11T17:18:46.494319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518502293637575:2067] [node 5] ICP01 ready to work
2025-11-11T17:18:46.501726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518497516911587:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(35558223ns) {Configure:104(35554580ns) {ProcessConfigured:135(54736ns) {StartInitialHandshake:157(52998ns) {DropHandshakes:469(1097ns) {DropIncomingHandshake:441(333ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:476(3596ns) {SwitchToState:178(762ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.541906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518498253667189:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(45017160ns) {Configure:104(45013904ns) {ProcessConfigured:135(11943175ns) {StartInitialHandshake:157(11940215ns) {DropHandshakes:469(1703ns) {DropIncomingHandshake:441(283ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:476(5610ns) {SwitchToState:178(1110ns)}}}}}
2025-11-11T17:18:46.544390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518502548634550:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54068
2025-11-11T17:18:46.544569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518502548634551:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54068
2025-11-11T17:18:46.544848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518502548634562:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.554043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245480:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.526442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878954:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.569853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518502293637571:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(49429676ns) {Configure:104(49424225ns) {ProcessConfigured:135(48023564ns) {StartInitialHandshake:157(48021379ns) {DropHandshakes:469(1843ns) {DropIncomingHandshake:441(634ns) DropOutgoingHandshake:456(238ns)} GenerateSessionVirtualId:988(512ns) PrepareNewSessionHandshake:476(3331ns) {SwitchToState:178(2030ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.575841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518504121863136:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6247
2025-11-11T17:18:46.526649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878955:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
2025-11-11T17:18:46.609975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229406:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.613324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229407:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.617952Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518502493229406:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.618071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518498198262034:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.562004Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518502548634550:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.583585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518502293637576:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48189
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.593975Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518502548634551:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.526719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878956:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
2025-11-11T17:18:46.527813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518501811878954:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.594041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518501811878955:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.637994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878965:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
2025-11-11T17:18:46.638061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518497516911587:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.641715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229409:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.643879Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518502493229406:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:46.575967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518504121863137:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6247
2025-11-11T17:18:46.629872Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518500986245480:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.576047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518504121863138:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6247
2025-11-11T17:18:46.583737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518502293637577:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48189
2025-11-11T17:18:46.650966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518501811878956:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.594298Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518502548634562:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.594408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518498253667188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.634208Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518502548634557:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.634238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518502548634557:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.634854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518502548634553:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.634872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518502548634553:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.636205Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518502548634557:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.670356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878967:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
2025-11-11T17:18:46.670824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518501811878955:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:46.671051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518497516911587:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.671105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:46.671145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518497516911587:2048] [node 1] ICP22 created new session: [4:7571518501811878969:2048]
2025-11-11T17:18:46.671189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518501811878969:2048] [node 1] ICS09 handshake done sender: [4:7571518501811878955:2114] self: [4:7571518501811878966:0] peer: [1:7571518504121863127:0] socket: 168 qp: -1
2025-11-11T17:18:46.671220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518501811878969:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:46.671251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:46.630166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245481:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.644081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518498198262034:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.644152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:46.644192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518498198262034:2050] [node 3] ICP22 created new session: [6:7571518502493229410:2050]
2025-11-11T17:18:46.644228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518502493229410:2050] [node 3] ICS09 handshake done sender: [6:7571518502493229406:2115] self: [6:7571518502493229408:0] peer: [3:7571518502548634556:0] socket: 179 qp: -1
2025-11-11T17:18:46.644254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518502493229410:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:46.644286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:46.675176Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518502493229407:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.675267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518498198262033:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.605695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863136:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.583980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518502293637588:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48189
2025-11-11T17:18:46.584092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518502293637589:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48189
2025-11-11T17:18:46.601993Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637577:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.602107Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637588:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.603577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518497998670214:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.674554Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637576:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.681862Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637589:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.681979Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637579:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.682000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518502293637579:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.689811Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518502293637579:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.630308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518496691278108:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.638850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245483:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.682995Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518500986245481:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.697947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229414:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.613649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863137:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.637627Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863126:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.637663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518504121863126:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.642413Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863128:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.642434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518504121863128:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.654298Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518504121863126:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:46.665614Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518504121863128:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:46.681924Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863138:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.685720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518499826895762:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.685792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:46.685834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518499826895762:2049] [node 3] ICP22 created new session: [1:7571518504121863143:2049]
2025-11-11T17:18:46.685882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518504121863143:2049] [node 3] ICS09 handshake done sender: [1:7571518504121863126:2111] self: [1:7571518504121863125:0] peer: [3:7571518502548634567:0] socket: 161 qp: -1
2025-11-11T17:18:46.685913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518504121863143:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:46.685946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:46.686401Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863124:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.686420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518504121863124:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.686926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518504121863122:2076] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.686983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:46.687021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518504121863122:2076] [node 4] ICP22 created new session: [1:7571518504121863145:2076]
2025-11-11T17:18:46.687060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518504121863145:2076] [node 4] ICS09 handshake done sender: [1:7571518504121863128:2112] self: [1:7571518504121863127:0] peer: [4:7571518501811878966:0] socket: 162 qp: -1
2025-11-11T17:18:46.687089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518504121863145:2076] [node 4] ICS10 traffic start
2025-11-11T17:18:46.687112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:46.698241Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518502548634553:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:46.705725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518504121863124:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:46.706219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518499826895761:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.706259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:46.706302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518499826895761:2048] [node 2] ICP22 created new session: [1:7571518504121863151:2048]
2025-11-11T17:18:46.706344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518504121863151:2048] [node 2] ICS09 handshake done sender: [1:7571518504121863124:2110] self: [1:7571518504121863123:0] peer: [2:7571518502293637593:0] socket: 160 qp: -1
2025-11-11T17:18:46.706372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518504121863151:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:46.706406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:46.710023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518502548634548:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.710079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:46.710129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518502548634548:2063] [node 6] ICP22 created new session: [3:7571518502548634572:2063]
2025-11-11T17:18:46.710167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518502548634572:2063] [node 6] ICS09 handshake done sender: [3:7571518502548634557:2114] self: [3:7571518502548634556:0] peer: [6:7571518502493229408:0] socket: 176 qp: -1
2025-11-11T17:18:46.710202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518502548634572:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:46.710235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:46.710770Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.712319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245484:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.713089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518502293637571:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.713154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:46.713195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518502293637571:2064] [node 6] ICP22 created new session: [2:7571518502293637599:2064]
2025-11-11T17:18:46.713230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518502293637599:2064] [node 6] ICS09 handshake done sender: [2:7571518502293637579:2112] self: [2:7571518502293637578:0] peer: [6:7571518502493229413:0] socket: 186 qp: -1
2025-11-11T17:18:46.713261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518502293637599:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:46.713291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.721351Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518502493229407:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:46.714250Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518501811878965:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.714371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518497516911588:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.722015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518500986245484:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.722206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245485:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.722324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518496691278107:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.722395Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518500986245480:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:46.719676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637581:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.719707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518502293637581:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.720822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518502293637598:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48189
2025-11-11T17:18:46.722862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518502548634547:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.722924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:46.722967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518502548634547:2061] [node 5] ICP22 created new session: [3:7571518502548634574:2061]
2025-11-11T17:18:46.723002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518502548634574:2061] [node 5] ICS09 handshake done sender: [3:7571518502548634553:2112] self: [3:7571518502548634552:0] peer: [5:7571518500986245482:0] socket: 174 qp: -1
2025-11-11T17:18:46.723032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518502548634574:2061] [node 5] ICS10 traffic start
2025-11-11T17:18:46.723062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:46.723642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518498253667188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518502548634551:2111] held: no
2025-11-11T17:18:46.724353Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15804) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.726837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518498198262033:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.726912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:46.726948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518498198262033:2049] [node 2] ICP22 created new session: [6:7571518502493229416:2049]
2025-11-11T17:18:46.726987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518502493229416:2049] [node 2] ICS09 handshake done sender: [6:7571518502493229407:2116] self: [6:7571518502493229413:0] peer: [2:7571518502293637578:0] socket: 190 qp: -1
2025-11-11T17:18:46.727014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518502493229416:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:46.727046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:46.727212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518498198262033:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:46.727502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229415:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.725497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518496691278108:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.734210Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637583:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.734255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518502293637583:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.735487Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518502293637577:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.738095Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518502493229415:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.738311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518498198262032:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.735983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878973:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.743012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518502548634571:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.725570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:46.745878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863132:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.745919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518504121863132:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.746498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518504121863153:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6247
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.750070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.725617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518496691278108:2050] [node 3] ICP22 created new session: [5:7571518500986245488:2050]
2025-11-11T17:18:46.725654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518500986245488:2050] [node 3] ICS09 handshake done sender: [5:7571518500986245480:2115] self: [5:7571518500986245482:0] peer: [3:7571518502548634552:0] socket: 178 qp: -1
2025-11-11T17:18:46.725684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518500986245488:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:46.725713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:46.726194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245486:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.758678Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518502548634551:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.759164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
2025-11-11T17:18:46.759385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518497998670214:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.759441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:46.759526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518497998670214:2048] [node 1] ICP22 created new session: [2:7571518502293637604:2048]
2025-11-11T17:18:46.759571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518502293637604:2048] [node 1] ICS09 handshake done sender: [2:7571518502293637577:2111] self: [2:7571518502293637593:0] peer: [1:7571518504121863123:0] socket: 166 qp: -1
2025-11-11T17:18:46.759753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518498253667188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.759807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:46.759844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518498253667188:2048] [node 1] ICP22 created new session: [3:7571518502548634578:2048]
2025-11-11T17:18:46.760289Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518500986245485:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.760377Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518500986245486:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.757876Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518504121863132:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:46.758415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518504121863121:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.758475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:46.758514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518504121863121:2075] [node 6] ICP22 created new session: [1:7571518504121863155:2075]
2025-11-11T17:18:46.758558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518504121863155:2075] [node 6] ICS09 handshake done sender: [1:7571518504121863132:2114] self: [1:7571518504121863131:0] peer: [6:7571518502493229418:0] socket: 204 qp: -1
2025-11-11T17:18:46.758592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518504121863155:2075] [node 6] ICS10 traffic start
2025-11-11T17:18:46.758627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:46.762331Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:46.763777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518498198262032:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518502493229415:2119] held: no
2025-11-11T17:18:46.763901Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518502493229415:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:46.764170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518498198262032:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.764238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:46.764293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518498198262032:2048] [node 1] ICP22 created new session: [6:7571518502493229420:2048]
2025-11-11T17:18:46.764334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518502493229420:2048] [node 1] ICS09 handshake done sender: [6:7571518502493229415:2119] self: [6:7571518502493229418:0] peer: [1:7571518504121863131:0] socket: 206 qp: -1
2025-11-11T17:18:46.764357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518502493229420:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:46.764387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:46.760968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518500986245425:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.769727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518496691278106:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.777729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518502548634578:2048] [node 1] ICS09 handshake done sender: [3:7571518502548634551:2111] self: [3:7571518502548634567:0] peer: [1:7571518504121863125:0] socket: 163 qp: -1
2025-11-11T17:18:46.777919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245491:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.778509Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518501811878958:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.778537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518501811878958:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.780442Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518501811878958:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.781337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245495:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.759604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518502293637604:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:46.781895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518502548634578:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:46.781947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:46.782152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518498253667188:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22517247ns) {HandleHandshakeStatus:329(22330021ns) {DropIncomingHandshake:441(412ns) DropOutgoingHandshake:456(195ns) SwitchToState:178(155ns) ProcessPendingSessionEvents:495(941ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.794882Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863130:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.794922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518504121863130:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.796112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518504121863157:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6247
2025-11-11T17:18:46.796417Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518500986245485:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:46.759745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:46.800994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518497998670214:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(34331207ns) {HandleHandshakeStatus:329(34290066ns) {DropIncomingHandshake:441(398ns) DropOutgoingHandshake:456(3750ns) SwitchToState:178(405ns) ProcessPendingSessionEvents:495(33816057ns) {ForwardSessionEventToSession:612(35689ns) ForwardSessionEventToSession:612(5818888ns) ForwardSessionEventToSession:612(4707ns) ForwardSessionEventToSession:612(8054270ns) ForwardSessionEventToSession:612(8080523ns) ForwardSessionEventToSession:612(2735ns) ForwardSessionEventToSession:612(2799ns) ForwardSessionEventToSession:612(3213499ns) ForwardSessionEventToSession:612(3082ns) ForwardSessionEventToSession:612(8576766ns)}}}
2025-11-11T17:18:46.801937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518504121863130:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:46.805732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518500986245425:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.805810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:46.805853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518500986245425:2066] [node 4] ICP22 created new session: [5:7571518500986245497:2066]
2025-11-11T17:18:46.805896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518500986245497:2066] [node 4] ICS09 handshake done sender: [5:7571518500986245485:2119] self: [5:7571518500986245493:0] peer: [4:7571518501811878957:0] socket: 201 qp: -1
2025-11-11T17:18:46.805926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518500986245497:2066] [node 4] ICS10 traffic start
2025-11-11T17:18:46.805958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:46.801705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518501811878901:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.801855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:46.801916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518501811878901:2062] [node 5] ICP22 created new session: [4:7571518501811878976:2062]
2025-11-11T17:18:46.801965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518501811878976:2062] [node 5] ICS09 handshake done sender: [4:7571518501811878958:2116] self: [4:7571518501811878957:0] peer: [5:7571518500986245493:0] socket: 171 qp: -1
2025-11-11T17:18:46.801994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518501811878976:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:46.802025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.814110Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518502293637581:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:46.813922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518504121863157:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.814011Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:46.830915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518504121863120:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.830991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:46.831045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518504121863120:2074] [node 5] ICP22 created new session: [1:7571518504121863159:2074]
2025-11-11T17:18:46.831085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518504121863159:2074] [node 5] ICS09 handshake done sender: [1:7571518504121863130:2113] self: [1:7571518504121863129:0] peer: [5:7571518500986245494:0] socket: 182 qp: -1
2025-11-11T17:18:46.831120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518504121863159:2074] [node 5] ICS10 traffic start
2025-11-11T17:18:46.831180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:46.831700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518504121863120:2074] [node 5] ICP18 handshake (actor [1:7571518504121863157:2119]) is too old
2025-11-11T17:18:46.829650Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518502548634561:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.829714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518502548634561:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.832756Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518502548634561:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.814930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229422:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.811082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518501811878974:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.811377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518497516911589:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.834171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518500986245496:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21263
2025-11-11T17:18:46.838059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518502548634549:2069] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.838122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:46.838161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518502548634549:2069] [node 4] ICP22 created new session: [3:7571518502548634581:2069]
2025-11-11T17:18:46.838198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518502548634581:2069] [node 4] ICS09 handshake done sender: [3:7571518502548634561:2116] self: [3:7571518502548634560:0] peer: [4:7571518501811878979:0] socket: 212 qp: -1
2025-11-11T17:18:46.838230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518502548634581:2069] [node 4] ICS10 traffic start
2025-11-11T17:18:46.838264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:46.841320Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:46.846862Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:46.846475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518502293637574:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.846538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:46.846576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518502293637574:2065] [node 4] ICP22 created new session: [2:7571518502293637607:2065]
2025-11-11T17:18:46.846619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518502293637607:2065] [node 4] ICS09 handshake done sender: [2:7571518502293637581:2113] self: [2:7571518502293637580:0] peer: [4:7571518501811878972:0] socket: 187 qp: -1
2025-11-11T17:18:46.846652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518502293637607:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:46.846685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:46.848229Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518501811878965:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:46.853754Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518502293637583:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:46.858216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518496691278106:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518500986245486:2120] held: no
2025-11-11T17:18:46.877882Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518500986245486:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:46.879925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518502293637575:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.879991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:46.880032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518502293637575:2067] [node 5] ICP22 created new session: [2:7571518502293637609:2067]
2025-11-11T17:18:46.880067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518502293637609:2067] [node 5] ICS09 handshake done sender: [2:7571518502293637583:2114] self: [2:7571518502293637582:0] peer: [5:7571518500986245487:0] socket: 188 qp: -1
2025-11-11T17:18:46.880123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518502293637609:2067] [node 5] ICS10 traffic start
2025-11-11T17:18:46.880164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.880804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518502293637606:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48189
2025-11-11T17:18:46.877999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518497516911589:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518501811878974:2121] held: no
2025-11-11T17:18:46.878131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518497516911588:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.878179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:46.878222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518497516911588:2049] [node 2] ICP22 created new session: [4:7571518501811878981:2049]
2025-11-11T17:18:46.878261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518501811878981:2049] [node 2] ICS09 handshake done sender: [4:7571518501811878965:2118] self: [4:7571518501811878972:0] peer: [2:7571518502293637580:0] socket: 189 qp: -1
2025-11-11T17:18:46.878302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518501811878981:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:46.878332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:46.879090Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15804) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:46.889893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518501811878980:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10771
2025-11-11T17:18:46.890011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518496691278106:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.890071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:46.890113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518496691278106:2048] [node 1] ICP22 created new session: [5:7571518500986245500:2048]
2025-11-11T17:18:46.890157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518500986245500:2048] [node 1] ICS09 handshake done sender: [5:7571518500986245486:2120] self: [5:7571518500986245494:0] peer: [1:7571518504121863129:0] socket: 175 qp: -1
2025-11-11T17:18:46.890192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518500986245500:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:46.890221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.881248Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518502493229422:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.897840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518502548634583:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54068
2025-11-11T17:18:46.881368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518498198262046:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.893955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229424:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.918736Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15804) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:46.903950Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637606:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.920112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518502293637587:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.920137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518502293637587:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.917656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518502548634583:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.917844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518498253667189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.929774Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518502293637587:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:46.930130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518497998670215:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.930196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:46.930241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518497998670215:2049] [node 3] ICP22 created new session: [2:7571518502293637614:2049]
2025-11-11T17:18:46.930279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518502293637614:2049] [node 3] ICS09 handshake done sender: [2:7571518502293637587:2116] self: [2:7571518502293637586:0] peer: [3:7571518502548634584:0] socket: 165 qp: -1
2025-11-11T17:18:46.930310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518502293637614:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:46.930341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:46.918334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518497516911588:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:46.918552Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518501811878960:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.918571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518501811878960:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:46.935145Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518500986245481:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:46.935455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518496691278107:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15804) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518500986245481:2116] held: no
2025-11-11T17:18:46.935539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518496691278107:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.935581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:46.935615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518496691278107:2049] [node 2] ICP22 created new session: [5:7571518500986245502:2049]
2025-11-11T17:18:46.935651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518500986245502:2049] [node 2] ICS09 handshake done sender: [5:7571518500986245481:2116] self: [5:7571518500986245487:0] peer: [2:7571518502293637582:0] socket: 191 qp: -1
2025-11-11T17:18:46.938118Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518502493229424:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.938227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518502493229393:2080] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:46.935678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518500986245502:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:46.935709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:46.937645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518502548634585:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54068
2025-11-11T17:18:46.942045Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518501811878974:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:46.942544Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15804) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:46.945174Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518500986245443:2078] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:46.945207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518500986245443:2078] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:46.946406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229426:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.946455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518502493229427:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64824
2025-11-11T17:18:46.945982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518497516911589:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.946047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:46.946090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518497516911589:2050] [node 3] ICP22 created new session: [4:7571518501811878984:2050]
2025-11-11T17:18:46.957668Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518500986245443:2078] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:46.957926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518501811878984:2050] [node 3] ICS09 handshake done sender: [4:7571518501811878974:2121] self: [4:7571518501811878979:0] peer: [3:7571518502548634560:0] socket: 213 qp: -1
2025-11-11T17:18:46.958351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518501811878984:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:46.958392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:46.958189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518498253667189:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15804) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518502548634583:2119] held: no
2025-11-11T17:18:46.959032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518502548634583:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:46.961713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518500986245441:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.961775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:46.961811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518500986245441:2077] [node 6] ICP22 created new session: [5:7571518500986245505:2077]
2025-11-11T17:18:46.961845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518500986245505:2077] [node 6] ICS09 handshake done sender: [5:7571518500986245443:2078] self: [5:7571518500986245442:0] peer: [6:7571518502493229425:0] socket: 170 qp: -1
2025-11-11T17:18:46.961872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518500986245505:2077] [node 6] ICS10 traffic start
2025-11-11T17:18:46.961894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:46.961395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518498253667189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.961462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:46.961503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518498253667189:2049] [node 2] ICP22 created new session: [3:7571518502548634586:2049]
2025-11-11T17:18:46.961543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518502548634586:2049] [node 2] ICS09 handshake done sender: [3:7571518502548634583:2119] self: [3:7571518502548634584:0] peer: [2:7571518502293637586:0] socket: 172 qp: -1
2025-11-11T17:18:46.961570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518502548634586:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:46.961616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:46.965995Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518501811878960:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:46.959428Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518502493229424:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:46.959598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518502493229393:2080] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:46.959634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:46.959672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518502493229393:2080] [node 5] ICP22 created new session: [6:7571518502493229428:2080]
2025-11-11T17:18:46.959713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518502493229428:2080] [node 5] ICS09 handshake done sender: [6:7571518502493229424:2122] self: [6:7571518502493229425:0] peer: [5:7571518500986245442:0] socket: 177 qp: -1
2025-11-11T17:18:46.959745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518502493229428:2080] [node 5] ICS10 traffic start
2025-11-11T17:18:46.959785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:46.989952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518501811878944:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:46.990045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:46.990101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518501811878944:2063] [node 6] ICP22 created new session: [4:7571518501811878986:2063]
2025-11-11T17:18:46.990145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518501811878986:2063] [node 6] ICS09 handshake done sender: [4:7571518501811878960:2117] self: [4:7571518501811878959:0] peer: [6:7571518502493229423:0] socket: 159 qp: -1
2025-11-11T17:18:46.990180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518501811878986:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:46.990210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:47.009465Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:47.018043Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518502493229422:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:47.018336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518498198262046:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518502493229422:2121] held: no
2025-11-11T17:18:47.018429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518498198262046:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:47.018477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:47.018527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518498198262046:2059] [node 4] ICP22 created new session: [6:7571518506788196726:2059]
2025-11-11T17:18:47.018585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518506788196726:2059] [node 4] ICS09 handshake done sender: [6:7571518502493229422:2121] self: [6:7571518502493229423:0] peer: [4:7571518501811878959:0] socket: 167 qp: -1
2025-11-11T17:18:47.018620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518506788196726:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:47.018660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:47.051396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:47.051622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518502548634578:2048] [node 1] ICS01 socket: 163 reason# EndOfStream
2025-11-11T17:18:47.051664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518498253667188:2048] [node 1] ICP30 unregister session Session# [3:7571518502548634578:2048] VirtualId# [3:7571518502548634567:0]
2025-11-11T17:18:47.051693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518502548634578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.051751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:47.051809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:47.053818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:47.054007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518502293637604:2048] [node 1] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-11T17:18:47.054043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518502293637604:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.054130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:47.054170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518502293637604:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:47.057722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:47.057961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518500986245500:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-11T17:18:47.057999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518496691278106:2048] [node 1] ICP30 unregister session Session# [5:7571518500986245500:2048] VirtualId# [5:7571518500986245494:0]
2025-11-11T17:18:47.058030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518500986245500:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.058091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:47.058131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:47.065794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:47.066028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518502493229420:2048] [node 1] ICS01 socket: 206 reason# EndOfStream
2025-11-11T17:18:47.066072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518498198262032:2048] [node 1] ICP30 unregister session Session# [6:7571518502493229420:2048] VirtualId# [6:7571518502493229418:0]
2025-11-11T17:18:47.069745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:47.066108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518502493229420:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.066185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:47.066226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:47.058895Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Couldn't connect to any resolved address
2025-11-11T17:18:47.077785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518497998670214:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:47.077834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518502293637604:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:47.077867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518497998670214:2048] [node 1] ICP30 unregister session Session# [2:7571518502293637604:2048] VirtualId# [2:7571518502293637593:0]
2025-11-11T17:18:47.077911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:47.077942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518497998670214:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:47.077985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26392) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:47.069983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518501811878969:2048] [node 1] ICS01 socket: 168 reason# EndOfStream
2025-11-11T17:18:47.070020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518497516911587:2048] [node 1] ICP30 unregister session Session# [4:7571518501811878969:2048] VirtualId# [4:7571518501811878966:0]
2025-11-11T17:18:47.070050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518501811878969:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.070131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:47.070170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:47.093689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518497998670214:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:47.157762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:47.158073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518502548634586:2049] [node 2] ICS01 socket: 172 reason# EndOfStream
2025-11-11T17:18:47.158109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518498253667189:2049] [node 2] ICP30 unregister session Session# [3:7571518502548634586:2049] VirtualId# [3:7571518502548634584:0]
2025-11-11T17:18:47.158141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518502548634586:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.158237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:47.158282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:47.179157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:47.179336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518501811878981:2049] [node 2] ICS01 socket: 189 reason# EndOfStream
2025-11-11T17:18:47.179362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518497516911588:2049] [node 2] ICP30 unregister session Session# [4:7571518501811878981:2049] VirtualId# [4:7571518501811878972:0]
2025-11-11T17:18:47.179384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518501811878981:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.179428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:47.179454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:47.178304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:47.178535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518502493229416:2049] [node 2] ICS01 socket: 190 reason# EndOfStream
2025-11-11T17:18:47.178578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518498198262033:2049] [node 2] ICP30 unregister session Session# [6:7571518502493229416:2049] VirtualId# [6:7571518502493229413:0]
2025-11-11T17:18:47.178612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518502493229416:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.178688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:47.178731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:47.186047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:47.186257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518500986245502:2049] [node 2] ICS01 socket: 191 reason# EndOfStream
2025-11-11T17:18:47.186295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518496691278107:2049] [node 2] ICP30 unregister session Session# [5:7571518500986245502:2049] VirtualId# [5:7571518500986245487:0]
2025-11-11T17:18:47.186327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518500986245502:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.186414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:47.186455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:47.233812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:47.234035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518500986245488:2050] [node 3] ICS01 socket: 178 reason# EndOfStream
2025-11-11T17:18:47.234079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518496691278108:2050] [node 3] ICP30 unregister session Session# [5:7571518500986245488:2050] VirtualId# [5:7571518500986245482:0]
2025-11-11T17:18:47.234108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518500986245488:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.234198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:47.234241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:47.234327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:47.234501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518502493229410:2050] [node 3] ICS01 socket: 179 reason# EndOfStream
2025-11-11T17:18:47.234529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518498198262034:2050] [node 3] ICP30 unregister session Session# [6:7571518502493229410:2050] VirtualId# [6:7571518502493229408:0]
2025-11-11T17:18:47.234549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518502493229410:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.234601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:47.234631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:47.234706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:47.234873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518501811878984:2050] [node 3] ICS01 socket: 213 reason# EndOfStream
2025-11-11T17:18:47.234902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518497516911589:2050] [node 3] ICP30 unregister session Session# [4:7571518501811878984:2050] VirtualId# [4:7571518501811878979:0]
2025-11-11T17:18:47.234924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518501811878984:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.234966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:47.235000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:47.285917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:47.286195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518500986245497:2066] [node 4] ICS01 socket: 201 reason# EndOfStream
2025-11-11T17:18:47.286243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518500986245425:2066] [node 4] ICP30 unregister session Session# [5:7571518500986245497:2066] VirtualId# [5:7571518500986245493:0]
2025-11-11T17:18:47.286279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518500986245497:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.286371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:47.286415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:47.289685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:47.289916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518506788196726:2059] [node 4] ICS01 socket: 167 reason# EndOfStream
2025-11-11T17:18:47.289948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518498198262046:2059] [node 4] ICP30 unregister session Session# [6:7571518506788196726:2059] VirtualId# [6:7571518502493229423:0]
2025-11-11T17:18:47.289978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518506788196726:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.290065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:47.290108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:47.311848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:47.312055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518502493229428:2080] [node 5] ICS01 socket: 177 reason# EndOfStream
2025-11-11T17:18:47.312090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518502493229393:2080] [node 5] ICP30 unregister session Session# [6:7571518502493229428:2080] VirtualId# [6:7571518502493229425:0]
2025-11-11T17:18:47.312126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518502493229428:2080] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:47.312218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:47.312258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:47.453842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518505783740041:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:47.453881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518505783740042:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:47.537829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518508463168423:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:47.537886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518508463168424:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:47.625947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518508303174014:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:47.625990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518508303174015:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:47.717855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518508935203668:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:47.717898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518508935203669:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:47.717920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518508935203670:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:47.809519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518508830274560:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:47.809565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518508830274561:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:47.809608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518508830274562:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:47.986068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518505387647327:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:47.986106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518505387647328:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:47.986128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518505387647329:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:48.117813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518512598141336:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:48.118021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518512598141337:2066] [node 5] ICP01 ready to work
2025-11-11T17:18:48.118145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518512598141338:2068] [node 6] ICP01 ready to work
2025-11-11T17:18:48.123907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518508830274561:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(60348868ns) {Configure:104(60342554ns) {ProcessConfigured:135(4877068ns) {StartInitialHandshake:157(4873405ns) {DropHandshakes:469(3219ns) {DropIncomingHandshake:441(866ns) DropOutgoingHandshake:456(538ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:476(3184ns) {SwitchToState:178(1231ns)}}}}}
2025-11-11T17:18:48.117994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518513230170982:2063] [node 6] ICP01 ready to work
2025-11-11T17:18:48.110022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518510078707372:2076] [node 5] ICP01 ready to work
2025-11-11T17:18:48.118401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518512758135743:2067] [node 5] ICP01 ready to work
2025-11-11T17:18:48.168369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518513230170983:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.118670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518512758135744:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:48.117785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518510078707374:2077] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.200603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518513230170982:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(82297062ns) {Configure:104(82293420ns) {ProcessConfigured:135(75817ns) {StartInitialHandshake:157(72496ns) {DropHandshakes:469(3674ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(661ns)} GenerateSessionVirtualId:988(865ns) PrepareNewSessionHandshake:476(2140ns) {SwitchToState:178(625ns)}}}}}
2025-11-11T17:18:48.177253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518512598141349:2076] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61021
2025-11-11T17:18:48.158537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518513125241873:2060] [node 6] ICP01 ready to work
2025-11-11T17:18:48.218060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518508935203668:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17227253ns) {Configure:104(17222345ns) {ProcessConfigured:135(78935ns) {StartInitialHandshake:157(75100ns) {DropHandshakes:469(7994ns) {DropIncomingHandshake:441(631ns) DropOutgoingHandshake:456(456ns)} GenerateSessionVirtualId:988(960ns) PrepareNewSessionHandshake:476(2051ns) {SwitchToState:178(604ns)}}}}}
2025-11-11T17:18:48.123719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518512758135745:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:48.157471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518510078707376:2075] [node 4] ICP01 ready to work
2025-11-11T17:18:48.225977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518505783740041:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(107903358ns) {Configure:104(107898890ns) {ProcessConfigured:135(48000ns) {StartInitialHandshake:157(45462ns) {DropHandshakes:469(3064ns) {DropIncomingHandshake:441(542ns) DropOutgoingHandshake:456(852ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:476(1039ns) {SwitchToState:178(426ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.145991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518508463168424:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(22035980ns) {Configure:104(22032329ns) {ProcessConfigured:135(10238904ns) {StartInitialHandshake:157(10235629ns) {DropHandshakes:469(1293ns) {DropIncomingHandshake:441(339ns) DropOutgoingHandshake:456(199ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:476(5546ns) {SwitchToState:178(1881ns)}}}}}
2025-11-11T17:18:48.233848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518513125241873:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(33808820ns) {Configure:104(33802800ns) {ProcessConfigured:135(32551921ns) {StartInitialHandshake:157(32549675ns) {DropHandshakes:469(2934ns) {DropIncomingHandshake:441(677ns) DropOutgoingHandshake:456(531ns)} GenerateSessionVirtualId:988(577ns) PrepareNewSessionHandshake:476(2960ns) {SwitchToState:178(1009ns)}}}}}
2025-11-11T17:18:48.169396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518508463168423:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(50496195ns) {Configure:104(50493741ns) {ProcessConfigured:135(85374ns) {StartInitialHandshake:157(82894ns) {DropHandshakes:469(3134ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(667ns)} GenerateSessionVirtualId:988(969ns) PrepareNewSessionHandshake:476(2097ns) {SwitchToState:178(657ns)}}}}}
2025-11-11T17:18:48.186161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518512758135759:2076] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64538
2025-11-11T17:18:48.223609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518512758135743:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31595918ns) {Configure:104(31591511ns) {ProcessConfigured:135(30371687ns) {StartInitialHandshake:157(30369049ns) {DropHandshakes:469(3725ns) {DropIncomingHandshake:441(813ns) DropOutgoingHandshake:456(396ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:476(2116ns) {SwitchToState:178(566ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.262594Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518505783740042:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(84967228ns) {Configure:104(84961887ns) {ProcessConfigured:135(52895ns) {StartInitialHandshake:157(49503ns) {DropHandshakes:469(1478ns) {DropIncomingHandshake:441(485ns) DropOutgoingHandshake:456(356ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:476(719ns) {SwitchToState:178(337ns)}}}}}
2025-11-11T17:18:48.263034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518510078707383:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24097
2025-11-11T17:18:48.263262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518510078707384:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24097
2025-11-11T17:18:48.263370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518510078707385:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24097
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.266122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518512758135760:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64538
2025-11-11T17:18:48.234183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518512598141336:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(83542418ns) {Configure:104(83539460ns) {ProcessConfigured:135(44411106ns) {StartInitialHandshake:157(44408380ns) {DropHandshakes:469(3284ns) {DropIncomingHandshake:441(794ns) DropOutgoingHandshake:456(573ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:476(3338ns) {SwitchToState:178(1873ns)}}}}}
2025-11-11T17:18:48.241957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241874:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.264331Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707383:2087] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.282869Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518512758135747:2071] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.282916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518512758135747:2071] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.249963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518512598141349:2076] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.313621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518508935203669:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(93396073ns) {Configure:104(93389547ns) {ProcessConfigured:135(82900ns) {StartInitialHandshake:157(79863ns) {DropHandshakes:469(3489ns) {DropIncomingHandshake:441(796ns) DropOutgoingHandshake:456(844ns)} GenerateSessionVirtualId:988(1237ns) PrepareNewSessionHandshake:476(2915ns) {SwitchToState:178(1111ns)}}}}}
2025-11-11T17:18:48.281706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707384:2088] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.242336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241877:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.309689Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518512758135760:2077] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.282776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518513125241874:2062] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.251296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518508303174015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.337137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518508463168423:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.343637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518512758135759:2076] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.401957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230170996:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-11T17:18:48.296662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518508830274562:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.337675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518512598141342:2074] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.368765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518505387647340:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.397505Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518512758135747:2071] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:48.396815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707378:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.545670Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518513230170996:2070] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.362171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241881:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.337718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518512598141342:2074] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.475448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518505387647340:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(106389522ns) {Configure:104(106383910ns) {ProcessConfigured:135(46731ns) {StartInitialHandshake:157(44302ns) {DropHandshakes:469(3632ns) {DropIncomingHandshake:441(770ns) DropOutgoingHandshake:456(582ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:476(1973ns) {SwitchToState:178(766ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.475869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518509682614660:2076] [node 5] ICP01 ready to work
2025-11-11T17:18:48.396882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518510078707378:2084] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.445931Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518510078707378:2084] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:48.446178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518505783740041:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.446239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:48.446290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518505783740041:2048] [node 2] ICP22 created new session: [1:7571518510078707418:2048]
2025-11-11T17:18:48.446334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518510078707418:2048] [node 2] ICS09 handshake done sender: [1:7571518510078707378:2084] self: [1:7571518510078707377:0] peer: [2:7571518512758135761:0] socket: 180 qp: -1
2025-11-11T17:18:48.446361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518510078707418:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:48.446423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:48.533684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707385:2089] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.339007Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518512598141342:2074] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:48.551192Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518513230170989:2067] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.410032Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518513125241877:2064] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.570189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518505387647329:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(190454099ns) {Configure:104(190448412ns) {ProcessConfigured:135(64534ns) {StartInitialHandshake:157(62119ns) {DropHandshakes:469(2990ns) {DropIncomingHandshake:441(659ns) DropOutgoingHandshake:456(611ns)} GenerateSessionVirtualId:988(662ns) PrepareNewSessionHandshake:476(1809ns) {SwitchToState:178(711ns)}}}}}
2025-11-11T17:18:48.579879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614696:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.580005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614697:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.580086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614698:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.580162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614699:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.580241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614700:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.551230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518513230170989:2067] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.339369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518512598141337:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.339426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:48.339473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518512598141337:2066] [node 5] ICP22 created new session: [3:7571518512598141353:2066]
2025-11-11T17:18:48.339520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518512598141353:2066] [node 5] ICS09 handshake done sender: [3:7571518512598141342:2074] self: [3:7571518512598141341:0] peer: [5:7571518513125241879:0] socket: 164 qp: -1
2025-11-11T17:18:48.339554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518512598141353:2066] [node 5] ICS10 traffic start
2025-11-11T17:18:48.339594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:48.480840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518512598141378:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61021
2025-11-11T17:18:48.550116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518512598141386:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61021
2025-11-11T17:18:48.550461Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518512598141349:2076] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:48.570789Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518512598141386:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.570905Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518512598141350:2077] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.570925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518512598141350:2077] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.571544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518508303174015:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.571588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:48.571629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518508303174015:2049] [node 2] ICP22 created new session: [3:7571518512598141397:2049]
2025-11-11T17:18:48.571671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518512598141397:2049] [node 2] ICS09 handshake done sender: [3:7571518512598141349:2076] self: [3:7571518512598141351:0] peer: [2:7571518512758135746:0] socket: 170 qp: -1
2025-11-11T17:18:48.571704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518512598141397:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:48.571741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:48.582962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518508303174014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.410391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518513125241892:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:48.610306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518512598141401:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61021
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.572578Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.410479Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518513125241874:2062] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:48.581964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518510078707426:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24097
2025-11-11T17:18:48.625809Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707380:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.625831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518510078707380:2085] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.634130Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518510078707380:2085] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:48.634358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518510078707372:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.634404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:48.634448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518510078707372:2076] [node 5] ICP22 created new session: [1:7571518510078707428:2076]
2025-11-11T17:18:48.634497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518510078707428:2076] [node 5] ICS09 handshake done sender: [1:7571518510078707380:2085] self: [1:7571518510078707379:0] peer: [5:7571518513125241942:0] socket: 181 qp: -1
2025-11-11T17:18:48.634529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518510078707428:2076] [node 5] ICS10 traffic start
2025-11-11T17:18:48.634557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:48.638717Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707389:2091] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.638747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518510078707389:2091] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.559951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518508935203670:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.582335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518508463168423:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518512758135760:2077] held: no
2025-11-11T17:18:48.630184Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518512598141350:2077] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:48.618589Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518509682614700:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.625849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518505387647328:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(148548210ns) {Configure:104(148544874ns) {ProcessConfigured:135(23951655ns) {StartInitialHandshake:157(23949034ns) {DropHandshakes:469(9148029ns) {DropIncomingHandshake:441(799ns) DropOutgoingHandshake:456(1501ns)} GenerateSessionVirtualId:988(1360ns) PrepareNewSessionHandshake:476(3834ns) {SwitchToState:178(1462ns)}}}}}
2025-11-11T17:18:48.638804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518509682614696:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.660286Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518509682614699:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.665781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518509682614697:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.670125Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518510078707389:2091] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.683705Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518509682614698:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.684995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518509682614660:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.688536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518512598141336:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.688591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:48.688627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518512598141336:2064] [node 4] ICP22 created new session: [3:7571518512598141403:2064]
2025-11-11T17:18:48.560058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230171041:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.582648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518512758135803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64538
2025-11-11T17:18:48.410719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518508830274562:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.688671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518512598141403:2064] [node 4] ICS09 handshake done sender: [3:7571518512598141350:2077] self: [3:7571518512598141348:0] peer: [4:7571518513230171045:0] socket: 186 qp: -1
2025-11-11T17:18:48.704209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518505387647327:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.688707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518512598141403:2064] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.597929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518508463168424:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.701725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518505783740042:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.562934Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518513230170989:2067] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:48.410757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.719886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518505387647329:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.701819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:48.701864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518505783740042:2049] [node 3] ICP22 created new session: [1:7571518510078707432:2049]
2025-11-11T17:18:48.718300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518510078707432:2049] [node 3] ICS09 handshake done sender: [1:7571518510078707389:2091] self: [1:7571518510078707388:0] peer: [3:7571518512598141400:0] socket: 176 qp: -1
2025-11-11T17:18:48.725754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518510078707432:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:48.738002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707387:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.738026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518510078707387:2090] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.738514Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707426:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.738685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518510078707382:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.738701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518510078707382:2086] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.739709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518510078707433:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24097
2025-11-11T17:18:48.739924Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518510078707387:2090] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:48.745834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:48.746341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518505783740042:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(64464285ns) {HandleHandshakeStatus:329(64460593ns) {DropIncomingHandshake:441(24783ns) DropOutgoingHandshake:456(614ns) SwitchToState:178(723ns) ProcessPendingSessionEvents:495(98859ns) {ForwardSessionEventToSession:612(55584ns) ForwardSessionEventToSession:612(3952ns) ForwardSessionEventToSession:612(6145ns) ForwardSessionEventToSession:612(4832ns) ForwardSessionEventToSession:612(5875ns) ForwardSessionEventToSession:612(2347ns) ForwardSessionEventToSession:612(2447ns) ForwardSessionEventToSession:612(4639ns) ForwardSessionEventToSession:612(1814ns) ForwardSessionEventToSession:612(1987ns)}}}
2025-11-11T17:18:48.746503Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518510078707382:2086] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:48.635497Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518513230171041:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.688740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:48.741706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518505387647328:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.597973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:48.410795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518508830274562:2050] [node 3] ICP22 created new session: [5:7571518513125241893:2050]
2025-11-11T17:18:48.410833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518513125241893:2050] [node 3] ICS09 handshake done sender: [5:7571518513125241874:2062] self: [5:7571518513125241879:0] peer: [3:7571518512598141341:0] socket: 166 qp: -1
2025-11-11T17:18:48.410865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518513125241893:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:48.410897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:48.434907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518513125241892:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23459525ns) {Configure:104(23453837ns) {ProcessConfigured:135(54060ns) {StartInitialHandshake:157(50877ns) {DropHandshakes:469(3396ns) {DropIncomingHandshake:441(643ns) DropOutgoingHandshake:456(687ns)} GenerateSessionVirtualId:988(1161ns) PrepareNewSessionHandshake:476(1985ns) {SwitchToState:178(686ns)}}}}}
2025-11-11T17:18:48.436319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518513125241892:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.613909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241880:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.618087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241940:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.621698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241941:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.621895Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518513125241880:2065] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.622018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518508830274560:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.633650Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518513125241877:2064] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:48.638223Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518513125241941:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.638314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518508830274561:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.663032Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:48.682158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518513125241892:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.682214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:48.682264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518513125241892:2070] [node 4] ICP22 created new session: [5:7571518513125241946:2070]
2025-11-11T17:18:48.682311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518513125241946:2070] [node 4] ICS09 handshake done sender: [5:7571518513125241877:2064] self: [5:7571518513125241899:0] peer: [4:7571518513230170988:0] socket: 178 qp: -1
2025-11-11T17:18:48.682345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518513125241946:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:48.682379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:48.682625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518513125241892:2070] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:48.683405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241944:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.698110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241945:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.718093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518513125241880:2065] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:48.733221Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518513125241876:2063] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.733248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518513125241876:2063] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.757654Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:48.765748Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518505387647328:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(21753012ns) {IncomingHandshake:261(12073328ns) {DropIncomingHandshake:441(613ns) PrepareNewSessionHandshake:476(8097171ns) {SwitchToState:178(3529ns)} GenerateSessionVirtualId:988(925ns) IssueIncomingHandshakeReply:193(44122ns) {SwitchToState:178(447ns)}}}
2025-11-11T17:18:48.777096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518508830274561:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518513125241941:2119] held: no
2025-11-11T17:18:48.635534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518513230170983:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.635562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:48.635601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518513230170983:2064] [node 5] ICP22 created new session: [4:7571518513230171046:2064]
2025-11-11T17:18:48.635661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518513230171046:2064] [node 5] ICS09 handshake done sender: [4:7571518513230170989:2067] self: [4:7571518513230170988:0] peer: [5:7571518513125241899:0] socket: 174 qp: -1
2025-11-11T17:18:48.635688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518513230171046:2064] [node 5] ICS10 traffic start
2025-11-11T17:18:48.635738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:48.636210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518513230170983:2064] [node 5] ICP18 handshake (actor [4:7571518513230171041:2115]) is too old
2025-11-11T17:18:48.650452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230171049:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-11T17:18:48.659738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230171051:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-11T17:18:48.659886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230171052:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-11T17:18:48.659952Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518513230170996:2070] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:48.669641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230171050:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-11T17:18:48.705897Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518513230171051:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.705942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518508935203670:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.705986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:48.706039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518508935203670:2050] [node 3] ICP22 created new session: [4:7571518513230171053:2050]
2025-11-11T17:18:48.706077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518513230171053:2050] [node 3] ICS09 handshake done sender: [4:7571518513230170996:2070] self: [4:7571518513230171045:0] peer: [3:7571518512598141348:0] socket: 187 qp: -1
2025-11-11T17:18:48.706112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518513230171053:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:48.706149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:48.707079Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518513230171052:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.707550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518508935203669:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.707621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518508935203668:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.746638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230171057:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-11T17:18:48.757701Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:48.774684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518508935203669:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518513230171051:2119] held: no
2025-11-11T17:18:48.774898Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518513230171050:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.774989Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:48.781715Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518513230171051:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:48.777815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518505387647340:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:48.598020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518508463168424:2049] [node 3] ICP22 created new session: [2:7571518512758135805:2049]
2025-11-11T17:18:48.689449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518512598141402:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61021
2025-11-11T17:18:48.770181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518510078707376:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.770260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:48.770305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518510078707376:2075] [node 4] ICP22 created new session: [1:7571518510078707438:2075]
2025-11-11T17:18:48.770353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518510078707438:2075] [node 4] ICS09 handshake done sender: [1:7571518510078707387:2090] self: [1:7571518510078707386:0] peer: [4:7571518513230171056:0] socket: 213 qp: -1
2025-11-11T17:18:48.770386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518510078707438:2075] [node 4] ICS10 traffic start
2025-11-11T17:18:48.770462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:48.783225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518510078707374:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.783279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:48.783320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518510078707374:2077] [node 6] ICP22 created new session: [1:7571518510078707440:2077]
2025-11-11T17:18:48.783360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518510078707440:2077] [node 6] ICS09 handshake done sender: [1:7571518510078707382:2086] self: [1:7571518510078707381:0] peer: [6:7571518509682614707:0] socket: 182 qp: -1
2025-11-11T17:18:48.783393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518510078707440:2077] [node 6] ICS10 traffic start
2025-11-11T17:18:48.783430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:48.786301Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518513125241941:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:48.710968Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518512598141401:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.711070Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518512598141386:2108] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:48.751065Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:48.753716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518508303174014:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.761777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:48.761839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518508303174014:2048] [node 1] ICP22 created new session: [3:7571518512598141406:2048]
2025-11-11T17:18:48.773678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518512598141406:2048] [node 1] ICS09 handshake done sender: [3:7571518512598141386:2108] self: [3:7571518512598141400:0] peer: [1:7571518510078707388:0] socket: 193 qp: -1
2025-11-11T17:18:48.773746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518512598141406:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:48.773808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:48.776073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518512598141344:2075] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.776096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518512598141344:2075] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.777249Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518512598141344:2075] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:48.777435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518512598141338:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.777481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:48.777516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518512598141338:2068] [node 6] ICP22 created new session: [3:7571518512598141409:2068]
2025-11-11T17:18:48.777570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518512598141409:2068] [node 6] ICS09 handshake done sender: [3:7571518512598141344:2075] self: [3:7571518512598141343:0] peer: [6:7571518509682614708:0] socket: 165 qp: -1
2025-11-11T17:18:48.777614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518512598141409:2068] [node 6] ICS10 traffic start
2025-11-11T17:18:48.777647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:48.785969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518508303174014:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32077859ns) {HandleHandshakeStatus:329(32073466ns) {DropIncomingHandshake:441(710ns) DropOutgoingHandshake:456(5563ns) SwitchToState:178(661ns) ProcessPendingSessionEvents:495(11851887ns) {ForwardSessionEventToSession:612(11789667ns) ForwardSessionEventToSession:612(7147ns) ForwardSessionEventToSession:612(4881ns) ForwardSessionEventToSession:612(2214ns) ForwardSessionEventToSession:612(7998ns) ForwardSessionEventToSession:612(4820ns) ForwardSessionEventToSession:612(2601ns) ForwardSessionEventToSession:612(4200ns) ForwardSessionEventToSession:612(2998ns) ForwardSessionEventToSession:612(2530ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.598063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518512758135805:2049] [node 3] ICS09 handshake done sender: [2:7571518512758135747:2071] self: [2:7571518512758135746:0] peer: [3:7571518512598141351:0] socket: 167 qp: -1
2025-11-11T17:18:48.598099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518512758135805:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:48.598133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:48.606197Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518512758135760:2077] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:48.608971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518508463168423:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.609024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:48.609070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518508463168423:2048] [node 1] ICP22 created new session: [2:7571518512758135808:2048]
2025-11-11T17:18:48.609111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518512758135808:2048] [node 1] ICS09 handshake done sender: [2:7571518512758135760:2077] self: [2:7571518512758135761:0] peer: [1:7571518510078707377:0] socket: 183 qp: -1
2025-11-11T17:18:48.609142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518512758135808:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:48.609172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:48.637990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518512758135810:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64538
2025-11-11T17:18:48.657752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518512758135758:2075] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.657778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518512758135758:2075] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.666118Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518512758135758:2075] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:48.704502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518512758135743:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.704550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:48.709762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518512758135743:2067] [node 5] ICP22 created new session: [2:7571518512758135812:2067]
2025-11-11T17:18:48.709815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518512758135812:2067] [node 5] ICS09 handshake done sender: [2:7571518512758135758:2075] self: [2:7571518512758135748:0] peer: [5:7571518513125241943:0] socket: 208 qp: -1
2025-11-11T17:18:48.709855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518512758135812:2067] [node 5] ICS10 traffic start
2025-11-11T17:18:48.709894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:48.711440Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518512758135810:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.711536Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518512758135754:2074] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.711556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518512758135754:2074] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.720054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518512758135754:2074] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:48.729979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518512758135744:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.730063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:48.730113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518512758135744:2070] [node 4] ICP22 created new session: [2:7571518512758135816:2070]
2025-11-11T17:18:48.730152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518512758135816:2070] [node 4] ICS09 handshake done sender: [2:7571518512758135754:2074] self: [2:7571518512758135753:0] peer: [4:7571518513230171055:0] socket: 207 qp: -1
2025-11-11T17:18:48.730189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518512758135816:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:48.730242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:48.805667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518512758135752:2073] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.805703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518512758135752:2073] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.806920Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518512758135752:2073] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:48.807150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518512758135745:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.807193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:48.807237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518512758135745:2064] [node 6] ICP22 created new session: [2:7571518512758135819:2064]
2025-11-11T17:18:48.807277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518512758135819:2064] [node 6] ICS09 handshake done sender: [2:7571518512758135752:2073] self: [2:7571518512758135751:0] peer: [6:7571518509682614709:0] socket: 169 qp: -1
2025-11-11T17:18:48.807309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518512758135819:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:48.807345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:48.798037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518508935203668:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518513230171052:2120] held: no
2025-11-11T17:18:48.818244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518512598141338:2068] [node 6] ICP35 event processing took too much time PendingConnection:278(40585562ns) {HandleHandshakeStatus:329(40578542ns) {DropIncomingHandshake:441(9535ns) DropOutgoingHandshake:456(167ns) SwitchToState:178(774ns) ProcessPendingSessionEvents:495(20281441ns) {ForwardSessionEventToSession:612(20226411ns) ForwardSessionEventToSession:612(5802ns) ForwardSessionEventToSession:612(7617ns) ForwardSessionEventToSession:612(7714ns) ForwardSessionEventToSession:612(3141ns) ForwardSessionEventToSession:612(2756ns) ForwardSessionEventToSession:612(5625ns) ForwardSessionEventToSession:612(2347ns) ForwardSessionEventToSession:612(5609ns) ForwardSessionEventToSession:612(2493ns)}}}
2025-11-11T17:18:48.789656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518508830274560:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.821665Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:48.798246Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518513230170985:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:48.825798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:48.818377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518508303174014:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:48.834072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518508830274560:2048] [node 1] ICP22 created new session: [5:7571518513125241950:2048]
2025-11-11T17:18:48.838003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518508830274561:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.838051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:48.843384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518513125241950:2048] [node 1] ICS09 handshake done sender: [5:7571518513125241880:2065] self: [5:7571518513125241942:0] peer: [1:7571518510078707379:0] socket: 184 qp: -1
2025-11-11T17:18:48.846258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518508830274561:2049] [node 2] ICP22 created new session: [5:7571518513125241951:2049]
2025-11-11T17:18:48.857781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518513125241950:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:48.857833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:48.858350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518508830274560:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(80407437ns) {HandleHandshakeStatus:329(68511545ns) {DropIncomingHandshake:441(1319ns) DropOutgoingHandshake:456(6659ns) SwitchToState:178(524ns) ProcessPendingSessionEvents:495(116725ns) {ForwardSessionEventToSession:612(50660ns) ForwardSessionEventToSession:612(6624ns) ForwardSessionEventToSession:612(2552ns) ForwardSessionEventToSession:612(2088ns) ForwardSessionEventToSession:612(4075ns) ForwardSessionEventToSession:612(3249ns) ForwardSessionEventToSession:612(24814ns) ForwardSessionEventToSession:612(2188ns) ForwardSessionEventToSession:612(2229ns) ForwardSessionEventToSession:612(9256ns)}}}
2025-11-11T17:18:48.877759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518513125241951:2049] [node 2] ICS09 handshake done sender: [5:7571518513125241941:2119] self: [5:7571518513125241943:0] peer: [2:7571518512758135748:0] socket: 210 qp: -1
2025-11-11T17:18:48.883072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518513125241951:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:48.883121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:48.841655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614711:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.798268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518513230170985:2065] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:48.798995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518508935203669:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.799040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:48.799081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518508935203669:2049] [node 2] ICP22 created new session: [4:7571518513230171059:2049]
2025-11-11T17:18:48.799118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518513230171059:2049] [node 2] ICS09 handshake done sender: [4:7571518513230171051:2119] self: [4:7571518513230171055:0] peer: [2:7571518512758135753:0] socket: 216 qp: -1
2025-11-11T17:18:48.799161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518513230171059:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:48.799192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:48.817780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518513230171058:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-11T17:18:48.820143Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518513230170985:2065] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:48.820251Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518513230171052:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:48.820414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518508935203668:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.820457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:48.820502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518508935203668:2048] [node 1] ICP22 created new session: [4:7571518513230171063:2048]
2025-11-11T17:18:48.820542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518513230171063:2048] [node 1] ICS09 handshake done sender: [4:7571518513230171052:2120] self: [4:7571518513230171056:0] peer: [1:7571518510078707386:0] socket: 217 qp: -1
2025-11-11T17:18:48.820570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518513230171063:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:48.820605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:48.821202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518513230170982:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.821250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:48.821289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518513230170982:2063] [node 6] ICP22 created new session: [4:7571518513230171065:2063]
2025-11-11T17:18:48.821332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518513230171065:2063] [node 6] ICS09 handshake done sender: [4:7571518513230170985:2065] self: [4:7571518513230170984:0] peer: [6:7571518509682614710:0] socket: 172 qp: -1
2025-11-11T17:18:48.821364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518513230171065:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:48.821388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:48.882751Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:48.881891Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24046) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:48.895679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518505387647327:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518509682614700:2118] held: no
2025-11-11T17:18:48.895875Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.896961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518505387647329:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24046) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518509682614696:2114] held: no
2025-11-11T17:18:48.897454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614712:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.900217Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518509682614700:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.900448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518505387647340:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518509682614698:2116] held: no
2025-11-11T17:18:48.900678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518505387647327:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.900734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:48.900771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518505387647327:2048] [node 1] ICP22 created new session: [6:7571518509682614716:2048]
2025-11-11T17:18:48.900805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518509682614716:2048] [node 1] ICS09 handshake done sender: [6:7571518509682614700:2118] self: [6:7571518509682614707:0] peer: [1:7571518510078707381:0] socket: 202 qp: -1
2025-11-11T17:18:48.900831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518509682614716:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:48.900878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:48.901301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614713:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.898569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518508830274561:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(60109674ns) {HandleHandshakeStatus:329(56090817ns) {DropIncomingHandshake:441(936ns) DropOutgoingHandshake:456(362ns) SwitchToState:178(953ns) ProcessPendingSessionEvents:495(1428ns)}}
2025-11-11T17:18:48.898705Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518513125241876:2063] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:48.899284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518513125241873:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:48.899318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:48.899355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518513125241873:2060] [node 6] ICP22 created new session: [5:7571518513125241954:2060]
2025-11-11T17:18:48.899394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518513125241954:2060] [node 6] ICS09 handshake done sender: [5:7571518513125241876:2063] self: [5:7571518513125241875:0] peer: [6:7571518509682614706:0] socket: 179 qp: -1
2025-11-11T17:18:48.899429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518513125241954:2060] [node 6] ICS10 traffic start
2025-11-11T17:18:48.899457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.909986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614714:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.913814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518509682614715:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60654
2025-11-11T17:18:48.914316Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518509682614696:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.915063Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518509682614697:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:48.915691Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518509682614698:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:48.915897Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518509682614699:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:48.915408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518513125241956:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35629
2025-11-11T17:18:48.916114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518505387647329:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.916177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:48.916220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518505387647329:2050] [node 3] ICP22 created new session: [6:7571518509682614720:2050]
2025-11-11T17:18:48.916256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518509682614720:2050] [node 3] ICS09 handshake done sender: [6:7571518509682614696:2114] self: [6:7571518509682614708:0] peer: [3:7571518512598141343:0] socket: 198 qp: -1
2025-11-11T17:18:48.916284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518509682614720:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:48.916334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:48.916638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518505387647340:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.916682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:48.916720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518505387647340:2059] [node 4] ICP22 created new session: [6:7571518509682614722:2059]
2025-11-11T17:18:48.916757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518509682614722:2059] [node 4] ICS09 handshake done sender: [6:7571518509682614698:2116] self: [6:7571518509682614710:0] peer: [4:7571518513230170984:0] socket: 200 qp: -1
2025-11-11T17:18:48.916782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518509682614722:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:48.916827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:48.917077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518509682614660:2076] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.917136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:48.917163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518509682614660:2076] [node 5] ICP22 created new session: [6:7571518509682614724:2076]
2025-11-11T17:18:48.917194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518509682614724:2076] [node 5] ICS09 handshake done sender: [6:7571518509682614699:2117] self: [6:7571518509682614706:0] peer: [5:7571518513125241875:0] socket: 201 qp: -1
2025-11-11T17:18:48.917220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518509682614724:2076] [node 5] ICS10 traffic start
2025-11-11T17:18:48.917256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:48.918144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518512758135821:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64538
2025-11-11T17:18:48.920425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518505387647328:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:48.920475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:48.920517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518505387647328:2049] [node 2] ICP22 created new session: [6:7571518509682614726:2049]
2025-11-11T17:18:48.920554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518509682614726:2049] [node 2] ICS09 handshake done sender: [6:7571518509682614697:2115] self: [6:7571518509682614709:0] peer: [2:7571518512758135751:0] socket: 199 qp: -1
2025-11-11T17:18:48.920584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518509682614726:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:48.920643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:48.921865Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:18:48.934358Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:48.934717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518512758135745:2064] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:48.965440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:48.965708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518512758135808:2048] [node 1] ICS01 socket: 183 reason# EndOfStream
2025-11-11T17:18:48.965749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518508463168423:2048] [node 1] ICP30 unregister session Session# [2:7571518512758135808:2048] VirtualId# [2:7571518512758135761:0]
2025-11-11T17:18:48.965776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518512758135808:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:48.965849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:48.965886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:48.967328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:48.967531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518512598141406:2048] [node 1] ICS07 socket disconnect 193 reason# EndOfStream
2025-11-11T17:18:48.967550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518512598141406:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:48.967602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:48.967634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518512598141406:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:48.974069Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:18:48.974749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518508303174014:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:18:48.974784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518512598141406:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:48.975327Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address
2025-11-11T17:18:48.975489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518508303174014:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:48.975524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518512598141406:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:48.975556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518508303174014:2048] [node 1] ICP30 unregister session Session# [3:7571518512598141406:2048] VirtualId# [3:7571518512598141400:0]
2025-11-11T17:18:48.975597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:48.975622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518508303174014:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:48.975634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:48.990804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:48.991065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518509682614716:2048] [node 1] ICS01 socket: 202 reason# EndOfStream
2025-11-11T17:18:48.991156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518505387647327:2048] [node 1] ICP30 unregister session Session# [6:7571518509682614716:2048] VirtualId# [6:7571518509682614707:0]
2025-11-11T17:18:48.991206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518509682614716:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:48.991338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:48.991433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:48.989727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518508303174014:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:48.992044Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:48.992350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518513230171063:2048] [node 1] ICS01 socket: 217 reason# EndOfStream
2025-11-11T17:18:48.992402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518508935203668:2048] [node 1] ICP30 unregister session Session# [4:7571518513230171063:2048] VirtualId# [4:7571518513230171056:0]
2025-11-11T17:18:48.992443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518513230171063:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:48.992545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:48.992617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:48.989759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:48.989977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518513125241950:2048] [node 1] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-11T17:18:48.990006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518513125241950:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:48.990103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:48.990150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518513125241950:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:49.006180Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address
2025-11-11T17:18:49.006517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518508830274560:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:49.006571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518513125241950:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:49.006606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518508830274560:2048] [node 1] ICP30 unregister session Session# [5:7571518513125241950:2048] VirtualId# [5:7571518513125241942:0]
2025-11-11T17:18:49.006662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:49.006689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518508830274560:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:49.006707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8542) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:49.022998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518508830274560:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:49.068176Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:49.068447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518512598141397:2049] [node 2] ICS01 socket: 170 reason# EndOfStream
2025-11-11T17:18:49.068489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518508303174015:2049] [node 2] ICP30 unregister session Session# [3:7571518512598141397:2049] VirtualId# [3:7571518512598141351:0]
2025-11-11T17:18:49.068528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518512598141397:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.068616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:49.068655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:49.070694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:49.070886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518509682614726:2049] [node 2] ICS01 socket: 199 reason# EndOfStream
2025-11-11T17:18:49.070920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518505387647328:2049] [node 2] ICP30 unregister session Session# [6:7571518509682614726:2049] VirtualId# [6:7571518509682614709:0]
2025-11-11T17:18:49.070947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518509682614726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.071053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:49.071096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:49.073711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:49.073711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:49.073916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518513125241951:2049] [node 2] ICS01 socket: 210 reason# EndOfStream
2025-11-11T17:18:49.073949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518508830274561:2049] [node 2] ICP30 unregister session Session# [5:7571518513125241951:2049] VirtualId# [5:7571518513125241943:0]
2025-11-11T17:18:49.073980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518513125241951:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.074060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:49.074105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:49.074533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518513230171059:2049] [node 2] ICS01 socket: 216 reason# EndOfStream
2025-11-11T17:18:49.074574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518508935203669:2049] [node 2] ICP30 unregister session Session# [4:7571518513230171059:2049] VirtualId# [4:7571518513230171055:0]
2025-11-11T17:18:49.074598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518513230171059:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.074661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:49.074698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:49.133784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:49.134008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518513125241893:2050] [node 3] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-11T17:18:49.134032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518513125241893:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.134121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:49.134168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518513125241893:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:49.135081Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24046) Couldn't connect to any resolved address
2025-11-11T17:18:49.135325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518508830274562:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24046) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:49.135368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518513125241893:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:49.135401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518508830274562:2050] [node 3] ICP30 unregister session Session# [5:7571518513125241893:2050] VirtualId# [5:7571518513125241879:0]
2025-11-11T17:18:49.135441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:49.135467Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518508830274562:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24046) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:49.135491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24046) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:49.137692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:49.137903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518509682614720:2050] [node 3] ICS01 socket: 198 reason# EndOfStream
2025-11-11T17:18:49.137939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518505387647329:2050] [node 3] ICP30 unregister session Session# [6:7571518509682614720:2050] VirtualId# [6:7571518509682614708:0]
2025-11-11T17:18:49.137971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518509682614720:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.138063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:49.138108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:49.141714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:49.141944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518513230171053:2050] [node 3] ICS01 socket: 187 reason# EndOfStream
2025-11-11T17:18:49.141985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518508935203670:2050] [node 3] ICP30 unregister session Session# [4:7571518513230171053:2050] VirtualId# [4:7571518513230171045:0]
2025-11-11T17:18:49.142011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518513230171053:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.142106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:49.142150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:49.145727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518508830274562:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:49.225698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:49.225951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518513125241946:2070] [node 4] ICS01 socket: 178 reason# EndOfStream
2025-11-11T17:18:49.225698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:49.225928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518509682614722:2059] [node 4] ICS01 socket: 200 reason# EndOfStream
2025-11-11T17:18:49.225981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518505387647340:2059] [node 4] ICP30 unregister session Session# [6:7571518509682614722:2059] VirtualId# [6:7571518509682614710:0]
2025-11-11T17:18:49.226011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518509682614722:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.226115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:49.226154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:49.225984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518513125241892:2070] [node 4] ICP30 unregister session Session# [5:7571518513125241946:2070] VirtualId# [5:7571518513125241899:0]
2025-11-11T17:18:49.226008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518513125241946:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.226111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:49.226152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:49.293764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:49.294010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518509682614724:2076] [node 5] ICS01 socket: 201 reason# EndOfStream
2025-11-11T17:18:49.294054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518509682614660:2076] [node 5] ICP30 unregister session Session# [6:7571518509682614724:2076] VirtualId# [6:7571518509682614706:0]
2025-11-11T17:18:49.294096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518509682614724:2076] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:49.294228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:49.294285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:49.509835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518514589339334:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:49.509877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518514589339335:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:49.573889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518516287499855:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:49.597739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518516287499854:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:49.641785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518516551145880:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:49.641823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518516551145881:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:49.685888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518514101393669:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:49.685941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518514101393670:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:49.685970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518514101393671:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:49.774289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518517830024038:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:49.774345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518517830024039:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:49.774371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518517830024040:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:49.961657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518514101393685:2059] [node 5] ICP01 ready to work
2025-11-11T17:18:50.039196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518514101393669:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(149200388ns) {Configure:104(149193810ns) {ProcessConfigured:135(63737ns) {StartInitialHandshake:157(60566ns) {DropHandshakes:469(3461ns) {DropIncomingHandshake:441(1136ns) DropOutgoingHandshake:456(676ns)} GenerateSessionVirtualId:988(853ns) PrepareNewSessionHandshake:476(2242ns) {SwitchToState:178(525ns)}}}}}
2025-11-11T17:18:49.965610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518516287499871:2061] [node 4] ICP01 ready to work
2025-11-11T17:18:49.961717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518516551145894:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:50.034857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518517830024054:2059] [node 4] ICP01 ready to work
2025-11-11T17:18:50.018604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518514589339367:2065] [node 4] ICP01 ready to work
2025-11-11T17:18:50.131344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518517830024038:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(141287777ns) {Configure:104(141283955ns) {ProcessConfigured:135(69256ns) {StartInitialHandshake:157(66304ns) {DropHandshakes:469(3122ns) {DropIncomingHandshake:441(688ns) DropOutgoingHandshake:456(579ns)} GenerateSessionVirtualId:988(800ns) PrepareNewSessionHandshake:476(1892ns) {SwitchToState:178(636ns)}}}}}
2025-11-11T17:18:49.977509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518516551145897:2060] [node 5] ICP01 ready to work
2025-11-11T17:18:50.098193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518520582467213:2072] [node 6] ICP01 ready to work
2025-11-11T17:18:49.888362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518515169914924:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.190501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518518396360998:2073] [node 6] ICP01 ready to work
2025-11-11T17:18:50.153657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518518884306694:2064] [node 5] ICP01 ready to work
2025-11-11T17:18:49.888412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518515169914925:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:50.182576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518520582467215:2076] [node 5] ICP01 ready to work
2025-11-11T17:18:50.298938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518516551145897:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(308884260ns) {Configure:104(308877688ns) {ProcessConfigured:135(46705ns) {StartInitialHandshake:157(44338ns) {DropHandshakes:469(2771ns) {DropIncomingHandshake:441(678ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(369ns) PrepareNewSessionHandshake:476(1444ns) {SwitchToState:178(549ns)}}}}}
2025-11-11T17:18:50.214299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518522124991370:2062] [node 6] ICP01 ready to work
2025-11-11T17:18:50.295016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518517830024054:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(213109560ns) {Configure:104(213101627ns) {ProcessConfigured:135(56537ns) {StartInitialHandshake:157(53466ns) {DropHandshakes:469(3381ns) {DropIncomingHandshake:441(942ns) DropOutgoingHandshake:456(539ns)} GenerateSessionVirtualId:988(940ns) PrepareNewSessionHandshake:476(2106ns) {SwitchToState:178(607ns)}}}}}
2025-11-11T17:18:50.255773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518518884306695:2077] [node 6] ICP01 ready to work
2025-11-11T17:18:49.888439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518515169914926:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.309690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518520846113239:2070] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.345515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991409:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
2025-11-11T17:18:49.940626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518515169914937:2059] [node 5] ICP01 ready to work
2025-11-11T17:18:50.330162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518516287499855:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(436091595ns) {Configure:104(436088410ns) {ProcessConfigured:135(56681ns) {StartInitialHandshake:157(54696ns) {DropHandshakes:469(3477ns) {DropIncomingHandshake:441(886ns) DropOutgoingHandshake:456(620ns)} GenerateSessionVirtualId:988(957ns) PrepareNewSessionHandshake:476(2135ns) {SwitchToState:178(635ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.409491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361039:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
2025-11-11T17:18:50.358089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991410:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.362017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518520846113251:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44583
2025-11-11T17:18:50.208009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882236:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.333201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518520582467222:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17439
2025-11-11T17:18:50.406470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518514589339334:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(387567126ns) {Configure:104(387561928ns) {ProcessConfigured:135(68022ns) {StartInitialHandshake:157(65203ns) {DropHandshakes:469(8454ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(491ns)} GenerateSessionVirtualId:988(749ns) PrepareNewSessionHandshake:476(1863ns) {SwitchToState:178(470ns)}}}}}
2025-11-11T17:18:50.362174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518520846113252:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44583
2025-11-11T17:18:50.329663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518519464882284:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:50.333269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518520582467223:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17439
2025-11-11T17:18:50.408067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518518884306696:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19244
2025-11-11T17:18:50.408228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518518884306697:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19244
2025-11-11T17:18:50.408578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518518884306704:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19244
2025-11-11T17:18:50.418161Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306696:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.418332Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306697:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.454245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518514589339335:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(492200345ns) {Configure:104(492196582ns) {ProcessConfigured:135(80512ns) {StartInitialHandshake:157(77047ns) {DropHandshakes:469(3036ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(562ns)} GenerateSessionVirtualId:988(414ns) PrepareNewSessionHandshake:476(1530ns) {SwitchToState:178(617ns)}}}}}
2025-11-11T17:18:50.462663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518518396361002:2078] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.462714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518518396361002:2078] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.413850Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113251:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.429929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113252:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.430210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518520846113255:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44583
2025-11-11T17:18:50.430315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518516551145881:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.438027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518516551145894:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(476023884ns) {Configure:104(476020538ns) {ProcessConfigured:135(62524ns) {StartInitialHandshake:157(59996ns) {DropHandshakes:469(6000ns) {DropIncomingHandshake:441(462ns) DropOutgoingHandshake:456(499ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:476(1201ns) {SwitchToState:178(257ns)}}}}}
2025-11-11T17:18:50.465969Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113255:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.463965Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518522124991410:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.505710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306704:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.481816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518518396361002:2078] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:50.389896Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467223:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.396350Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518519464882236:2062] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.499974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991412:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
2025-11-11T17:18:50.391520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518520582467227:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17439
2025-11-11T17:18:50.421782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467222:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.422240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518520582467228:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17439
2025-11-11T17:18:50.422287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518520582467229:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17439
2025-11-11T17:18:50.485748Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467217:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.485785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518520582467217:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.498427Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467219:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.498453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518520582467219:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.525783Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518520582467217:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:50.529714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518516287499854:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(572736045ns) {Configure:104(572731985ns) {ProcessConfigured:135(20809610ns) {StartInitialHandshake:157(20806779ns) {DropHandshakes:469(2669ns) {DropIncomingHandshake:441(469ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:476(3049ns) {SwitchToState:178(1524ns)}}}}}
2025-11-11T17:18:50.529944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518520582467219:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:50.418564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518515169914937:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(464627388ns) {Configure:104(464617680ns) {ProcessConfigured:135(56759ns) {StartInitialHandshake:157(54232ns) {DropHandshakes:469(3215ns) {DropIncomingHandshake:441(668ns) DropOutgoingHandshake:456(455ns)} GenerateSessionVirtualId:988(1359ns) PrepareNewSessionHandshake:476(1798ns) {SwitchToState:178(629ns)}}}}}
2025-11-11T17:18:50.499553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518514101393671:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(543869630ns) {Configure:104(543865157ns) {ProcessConfigured:135(4030551ns) {StartInitialHandshake:157(4028111ns) {DropHandshakes:469(3104ns) {DropIncomingHandshake:441(696ns) DropOutgoingHandshake:456(586ns)} GenerateSessionVirtualId:988(914ns) PrepareNewSessionHandshake:476(1100ns) {SwitchToState:178(426ns)}}}}}
2025-11-11T17:18:50.500137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518517830024039:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.561940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518516287499871:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(241788316ns) {Configure:104(241783128ns) {ProcessConfigured:135(64427ns) {StartInitialHandshake:157(60452ns) {DropHandshakes:469(3197ns) {DropIncomingHandshake:441(609ns) DropOutgoingHandshake:456(640ns)} GenerateSessionVirtualId:988(807ns) PrepareNewSessionHandshake:476(2263ns) {SwitchToState:178(664ns)}}}}}
2025-11-11T17:18:50.562247Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467221:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.562272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518520582467221:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.562936Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467228:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.563013Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467227:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.563508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518516287499854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.546279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518514101393685:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(531056666ns) {Configure:104(531046067ns) {ProcessConfigured:135(8412218ns) {StartInitialHandshake:157(8408513ns) {DropHandshakes:469(3244ns) {DropIncomingHandshake:441(677ns) DropOutgoingHandshake:456(503ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:476(3659ns) {SwitchToState:178(1583ns)}}}}}
2025-11-11T17:18:50.561807Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518518396361039:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.561853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518518396360998:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.561892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:50.561940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518518396360998:2073] [node 6] ICP22 created new session: [4:7571518518396361045:2073]
2025-11-11T17:18:50.561988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518518396361045:2073] [node 6] ICS09 handshake done sender: [4:7571518518396361002:2078] self: [4:7571518518396361001:0] peer: [6:7571518519464882296:0] socket: 173 qp: -1
2025-11-11T17:18:50.562036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518518396361045:2073] [node 6] ICS10 traffic start
2025-11-11T17:18:50.562069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.574011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518518884306694:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(248304589ns) {Configure:104(248299961ns) {ProcessConfigured:135(39903374ns) {StartInitialHandshake:157(39898674ns) {DropHandshakes:469(3243ns) {DropIncomingHandshake:441(666ns) DropOutgoingHandshake:456(603ns)} GenerateSessionVirtualId:988(888ns) PrepareNewSessionHandshake:476(4076ns) {SwitchToState:178(2431ns)}}}}}
2025-11-11T17:18:50.574138Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306699:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.574159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518518884306699:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.577003Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518518884306699:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:50.577297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518514589339367:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.577348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:50.577389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518514589339367:2065] [node 4] ICP22 created new session: [1:7571518518884306716:2065]
2025-11-11T17:18:50.577426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518518884306716:2065] [node 4] ICS09 handshake done sender: [1:7571518518884306699:2112] self: [1:7571518518884306698:0] peer: [4:7571518518396361048:0] socket: 192 qp: -1
2025-11-11T17:18:50.577459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518518884306716:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:50.577497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:50.418969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882286:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.597827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306701:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.597872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518518884306701:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.533724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518522124991409:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.565885Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113243:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.609990Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518518884306701:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:50.611383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518518884306720:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19244
2025-11-11T17:18:50.419129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518519464882284:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.570332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518514101393669:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.546634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518517830024040:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.599081Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518520582467221:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.565927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518520846113243:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.591445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361049:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
2025-11-11T17:18:50.592395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361050:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
2025-11-11T17:18:50.592673Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518518396361039:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:50.603954Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518518396361050:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.604002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518514101393669:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.604038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:50.604074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518514101393669:2048] [node 1] ICP22 created new session: [4:7571518518396361051:2048]
2025-11-11T17:18:50.604117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518518396361051:2048] [node 1] ICS09 handshake done sender: [4:7571518518396361039:2113] self: [4:7571518518396361048:0] peer: [1:7571518518884306698:0] socket: 195 qp: -1
2025-11-11T17:18:50.604150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518518396361051:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:50.604177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:50.637749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361053:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
2025-11-11T17:18:50.638096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361054:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
2025-11-11T17:18:50.599334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467229:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.419796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882295:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.654597Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518518396361053:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.655838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518514101393670:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.585727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518520846113261:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44583
2025-11-11T17:18:50.591878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518520846113262:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44583
2025-11-11T17:18:50.593091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113262:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.630939Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.631711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518516551145881:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518520846113252:2114] held: no
2025-11-11T17:18:50.631917Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518520846113252:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:50.632146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518516551145881:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.632183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:50.632225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518516551145881:2049] [node 2] ICP22 created new session: [3:7571518520846113265:2049]
2025-11-11T17:18:50.632263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518520846113265:2049] [node 2] ICS09 handshake done sender: [3:7571518520846113252:2114] self: [3:7571518520846113257:0] peer: [2:7571518520582467216:0] socket: 188 qp: -1
2025-11-11T17:18:50.632292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518520846113265:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:50.632320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:50.632650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518520846113264:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44583
2025-11-11T17:18:50.661746Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518520846113243:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:50.593827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518522124991412:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.462069Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518519464882286:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.623819Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.599438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518516287499855:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.665805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518514589339334:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.670075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518516551145897:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.670147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:50.670183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518516551145897:2060] [node 5] ICP22 created new session: [3:7571518520846113269:2060]
2025-11-11T17:18:50.670223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518520846113269:2060] [node 5] ICS09 handshake done sender: [3:7571518520846113243:2110] self: [3:7571518520846113242:0] peer: [5:7571518522124991414:0] socket: 176 qp: -1
2025-11-11T17:18:50.670261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518520846113269:2060] [node 5] ICS10 traffic start
2025-11-11T17:18:50.670298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:50.686380Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113264:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.462470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518515169914925:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.693929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518518396361054:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.599490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:50.599534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518516287499855:2049] [node 3] ICP22 created new session: [2:7571518520582467242:2049]
2025-11-11T17:18:50.599577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518520582467242:2049] [node 3] ICS09 handshake done sender: [2:7571518520582467217:2110] self: [2:7571518520582467216:0] peer: [3:7571518520846113257:0] socket: 181 qp: -1
2025-11-11T17:18:50.599607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518520582467242:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:50.599637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:50.600022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518520582467215:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.600079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:50.600114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518520582467215:2076] [node 5] ICP22 created new session: [2:7571518520582467244:2076]
2025-11-11T17:18:50.600160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518520582467244:2076] [node 5] ICS09 handshake done sender: [2:7571518520582467221:2112] self: [2:7571518520582467220:0] peer: [5:7571518522124991413:0] socket: 183 qp: -1
2025-11-11T17:18:50.600190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518520582467244:2076] [node 5] ICS10 traffic start
2025-11-11T17:18:50.600212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:50.601433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518520582467241:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17439
2025-11-11T17:18:50.629659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518520582467213:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.654293Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518520582467228:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:50.656507Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518520582467236:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.656529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518520582467236:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.656990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518516287499854:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.657067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:50.657117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518516287499854:2048] [node 1] ICP22 created new session: [2:7571518520582467250:2048]
2025-11-11T17:18:50.657160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518520582467250:2048] [node 1] ICS09 handshake done sender: [2:7571518520582467228:2116] self: [2:7571518520582467239:0] peer: [1:7571518518884306700:0] socket: 201 qp: -1
2025-11-11T17:18:50.657192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518520582467250:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:50.657223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:50.661629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:50.677640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518520582467213:2072] [node 6] ICP22 created new session: [2:7571518520582467249:2072]
2025-11-11T17:18:50.689671Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518520582467236:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:50.689758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518520582467249:2072] [node 6] ICS09 handshake done sender: [2:7571518520582467219:2111] self: [2:7571518520582467218:0] peer: [6:7571518519464882300:0] socket: 182 qp: -1
2025-11-11T17:18:50.689797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518520582467249:2072] [node 6] ICS10 traffic start
2025-11-11T17:18:50.689866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:50.690326Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518520582467213:2072] [node 6] ICP35 event processing took too much time PendingConnection:278(68462500ns) {HandleHandshakeStatus:329(68459400ns) {DropIncomingHandshake:441(26232ns) DropOutgoingHandshake:456(636ns) SwitchToState:178(590ns) ProcessPendingSessionEvents:495(97758ns) {ForwardSessionEventToSession:612(51359ns) ForwardSessionEventToSession:612(4924ns) ForwardSessionEventToSession:612(2564ns) ForwardSessionEventToSession:612(4360ns) ForwardSessionEventToSession:612(7456ns) ForwardSessionEventToSession:612(2586ns) ForwardSessionEventToSession:612(2799ns) ForwardSessionEventToSession:612(4574ns) ForwardSessionEventToSession:612(1841ns) ForwardSessionEventToSession:612(6276ns)}}}
2025-11-11T17:18:50.710143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518516287499871:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.710226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:50.710271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518516287499871:2061] [node 4] ICP22 created new session: [2:7571518520582467254:2061]
2025-11-11T17:18:50.710312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518520582467254:2061] [node 4] ICS09 handshake done sender: [2:7571518520582467236:2119] self: [2:7571518520582467235:0] peer: [4:7571518518396361056:0] socket: 168 qp: -1
2025-11-11T17:18:50.710349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518520582467254:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:50.710399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.681021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518517830024039:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518522124991410:2115] held: no
2025-11-11T17:18:50.694071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361057:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.478305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882301:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.674074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:50.478482Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518519464882295:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.738166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518514101393671:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.738279Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.753940Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518518396361053:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:50.729967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518516551145880:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.575578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882304:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.677768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518514589339367:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(28743945ns) {HandleHandshakeStatus:329(28741851ns) {DropIncomingHandshake:441(11222ns) DropOutgoingHandshake:456(636ns) SwitchToState:178(289ns) ProcessPendingSessionEvents:495(28377391ns) {ForwardSessionEventToSession:612(45009ns) ForwardSessionEventToSession:612(7026ns) ForwardSessionEventToSession:612(2287ns) ForwardSessionEventToSession:612(2375ns) ForwardSessionEventToSession:612(4305ns) ForwardSessionEventToSession:612(2573ns) ForwardSessionEventToSession:612(3638ns) ForwardSessionEventToSession:612(4298647ns) ForwardSessionEventToSession:612(2289ns) ForwardSessionEventToSession:612(23995363ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.681371Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.781946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518514101393670:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518518396361053:2118] held: no
2025-11-11T17:18:50.782044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518514101393670:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.782078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:50.782119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518514101393670:2049] [node 2] ICP22 created new session: [4:7571518518396361060:2049]
2025-11-11T17:18:50.782164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518518396361060:2049] [node 2] ICS09 handshake done sender: [4:7571518518396361053:2118] self: [4:7571518518396361056:0] peer: [2:7571518520582467235:0] socket: 175 qp: -1
2025-11-11T17:18:50.782189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518518396361060:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:50.782215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:50.782692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518518396360998:2073] [node 6] ICP18 handshake (actor [4:7571518518396361050:2117]) is too old
2025-11-11T17:18:50.783383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361059:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
2025-11-11T17:18:50.578561Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518519464882286:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:50.678164Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306720:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.752011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518520846113273:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44583
2025-11-11T17:18:50.678991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518518884306722:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19244
2025-11-11T17:18:50.699175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518517830024040:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518522124991409:2113] held: no
2025-11-11T17:18:50.578714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518515169914925:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.759782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518516551145880:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24286009ns) {IncomingHandshake:261(15851947ns) {DropIncomingHandshake:441(546ns) PrepareNewSessionHandshake:476(2689ns) {SwitchToState:178(1225ns)} GenerateSessionVirtualId:988(771ns) IssueIncomingHandshakeReply:193(9306639ns) {SwitchToState:178(1959ns)}}}
2025-11-11T17:18:50.822283Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518518396361059:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.699461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991418:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.680750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306722:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.578752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:50.774801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113259:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.719229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991420:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
2025-11-11T17:18:50.578792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518515169914925:2049] [node 2] ICP22 created new session: [6:7571518519464882305:2049]
2025-11-11T17:18:50.857657Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.682157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518514589339334:2048] [node 2] ICP22 created new session: [1:7571518518884306721:2048]
2025-11-11T17:18:50.719652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991415:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
2025-11-11T17:18:50.774838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518520846113259:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.786426Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518520846113254:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.786449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518520846113254:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.801128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518520846113259:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:50.802834Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518520846113254:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:50.805723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518516551145894:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.805777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:50.805830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518516551145894:2059] [node 6] ICP22 created new session: [3:7571518520846113276:2059]
2025-11-11T17:18:50.805871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518520846113276:2059] [node 6] ICS09 handshake done sender: [3:7571518520846113259:2117] self: [3:7571518520846113258:0] peer: [6:7571518519464882315:0] socket: 179 qp: -1
2025-11-11T17:18:50.805898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518520846113276:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:50.805927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:50.806993Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.809892Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518520846113264:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:50.816385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518520846113239:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.816450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:50.816490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518520846113239:2070] [node 4] ICP22 created new session: [3:7571518520846113278:2070]
2025-11-11T17:18:50.816537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518520846113278:2070] [node 4] ICS09 handshake done sender: [3:7571518520846113254:2115] self: [3:7571518520846113253:0] peer: [4:7571518518396361058:0] socket: 187 qp: -1
2025-11-11T17:18:50.816567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518520846113278:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:50.816598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:50.816963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518516551145880:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518520846113264:2120] held: no
2025-11-11T17:18:50.833934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518516551145880:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.834014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:50.834063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518516551145880:2048] [node 1] ICP22 created new session: [3:7571518520846113280:2048]
2025-11-11T17:18:50.834103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518520846113280:2048] [node 1] ICS09 handshake done sender: [3:7571518520846113264:2120] self: [3:7571518520846113272:0] peer: [1:7571518518884306709:0] socket: 220 qp: -1
2025-11-11T17:18:50.834137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518520846113280:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:50.834172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.578836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518519464882305:2049] [node 2] ICS09 handshake done sender: [6:7571518519464882286:2111] self: [6:7571518519464882300:0] peer: [2:7571518520582467218:0] socket: 190 qp: -1
2025-11-11T17:18:50.690970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306703:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.733643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
2025-11-11T17:18:50.738629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518522124991370:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(339816024ns) {Configure:104(339808810ns) {ProcessConfigured:135(19191073ns) {StartInitialHandshake:157(19187284ns) {DropHandshakes:469(3006ns) {DropIncomingHandshake:441(672ns) DropOutgoingHandshake:456(625ns)} GenerateSessionVirtualId:988(655ns) PrepareNewSessionHandshake:476(3177ns) {SwitchToState:178(1424ns)}}}}}
2025-11-11T17:18:50.741990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518522124991420:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.743271Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518522124991410:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:50.745899Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518522124991409:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:50.758068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518517830024038:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.774957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518517830024040:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.775009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:50.777675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518517830024039:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.777756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:50.777797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518517830024039:2049] [node 2] ICP22 created new session: [5:7571518522124991426:2049]
2025-11-11T17:18:50.777843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518522124991426:2049] [node 2] ICS09 handshake done sender: [5:7571518522124991410:2115] self: [5:7571518522124991413:0] peer: [2:7571518520582467220:0] socket: 185 qp: -1
2025-11-11T17:18:50.777884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518522124991426:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:50.777917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:50.778743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991428:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
2025-11-11T17:18:50.780939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518517830024040:2050] [node 3] ICP22 created new session: [5:7571518522124991425:2050]
2025-11-11T17:18:50.813768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518522124991425:2050] [node 3] ICS09 handshake done sender: [5:7571518522124991409:2113] self: [5:7571518522124991414:0] peer: [3:7571518520846113242:0] socket: 177 qp: -1
2025-11-11T17:18:50.817935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518522124991425:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:50.818529Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518522124991419:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.818834Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.819674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:50.822015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518517830024038:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518522124991420:2121] held: no
2025-11-11T17:18:50.822163Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518522124991420:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:50.857722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518517830024040:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(93107115ns) {HandleHandshakeStatus:329(93103812ns) {DropIncomingHandshake:441(2280ns) DropOutgoingHandshake:456(614ns) SwitchToState:178(608ns) ProcessPendingSessionEvents:495(1445ns)}}
2025-11-11T17:18:50.866102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518517830024038:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.866171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:50.866214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518517830024038:2048] [node 1] ICP22 created new session: [5:7571518522124991430:2048]
2025-11-11T17:18:50.866263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518522124991430:2048] [node 1] ICS09 handshake done sender: [5:7571518522124991420:2121] self: [5:7571518522124991424:0] peer: [1:7571518518884306712:0] socket: 228 qp: -1
2025-11-11T17:18:50.866304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518522124991430:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:50.866345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:50.867124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518517830024054:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.895289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518522124991433:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.865816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518514101393671:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518518396361054:2119] held: no
2025-11-11T17:18:50.873710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518518396361044:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.873738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518518396361044:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.885911Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518518396361044:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:50.909796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518518396361062:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56891
2025-11-11T17:18:50.910469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518514101393685:2059] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.910527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:50.910573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518514101393685:2059] [node 5] ICP22 created new session: [4:7571518518396361065:2059]
2025-11-11T17:18:50.910615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518518396361065:2059] [node 5] ICS09 handshake done sender: [4:7571518518396361044:2115] self: [4:7571518518396361043:0] peer: [5:7571518522124991432:0] socket: 202 qp: -1
2025-11-11T17:18:50.910651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518518396361065:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:50.910689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:50.912424Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518518396361054:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:50.913206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518514101393671:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.913264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:50.913300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518514101393671:2050] [node 3] ICP22 created new session: [4:7571518518396361067:2050]
2025-11-11T17:18:50.913340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518518396361067:2050] [node 3] ICS09 handshake done sender: [4:7571518518396361054:2119] self: [4:7571518518396361058:0] peer: [3:7571518520846113253:0] socket: 221 qp: -1
2025-11-11T17:18:50.913372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518518396361067:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:50.913408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:50.910277Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518522124991419:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:50.690991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518518884306703:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.578868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518519464882305:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:50.578893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:50.601712Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518519464882236:2062] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:50.626277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518519464882284:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.626342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:50.626391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518519464882284:2064] [node 4] ICP22 created new session: [6:7571518519464882308:2064]
2025-11-11T17:18:50.626441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518519464882308:2064] [node 4] ICS09 handshake done sender: [6:7571518519464882236:2062] self: [6:7571518519464882296:0] peer: [4:7571518518396361001:0] socket: 174 qp: -1
2025-11-11T17:18:50.626482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518519464882308:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:50.626519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:50.646520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518515169914924:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(334886691ns) {Configure:104(334882332ns) {ProcessConfigured:135(32300410ns) {StartInitialHandshake:157(32298423ns) {DropHandshakes:469(1776ns) {DropIncomingHandshake:441(191ns) DropOutgoingHandshake:456(522ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:476(1159ns) {SwitchToState:178(121ns)}}}}}
2025-11-11T17:18:50.694783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518515169914924:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.705954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882311:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.714076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518515169914924:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20829734ns) {IncomingHandshake:261(20826941ns) {DropIncomingHandshake:441(912ns) PrepareNewSessionHandshake:476(3891ns) {SwitchToState:178(2899ns)} GenerateSessionVirtualId:988(713ns) IssueIncomingHandshakeReply:193(9053081ns) {SwitchToState:178(2226ns)}}}
2025-11-11T17:18:50.720533Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518519464882311:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.737977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882314:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.753808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518515169914926:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.798000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518515169914926:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(24083124ns) {IncomingHandshake:261(15227437ns) {DropIncomingHandshake:441(1637ns) PrepareNewSessionHandshake:476(3883ns) {SwitchToState:178(2266ns)} GenerateSessionVirtualId:988(565ns) IssueIncomingHandshakeReply:193(7995396ns) {SwitchToState:178(2468ns)}}}
2025-11-11T17:18:50.817647Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.822920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518515169914924:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518519464882295:2116] held: no
2025-11-11T17:18:50.829972Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.854236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882313:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.890497Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518519464882314:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.894690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518515169914926:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518519464882311:2120] held: no
2025-11-11T17:18:50.894961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882316:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.897764Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518519464882295:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:50.921924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518515169914924:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.921988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:50.922030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518515169914924:2048] [node 1] ICP22 created new session: [6:7571518519464882317:2048]
2025-11-11T17:18:50.922073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518519464882317:2048] [node 1] ICS09 handshake done sender: [6:7571518519464882295:2116] self: [6:7571518519464882312:0] peer: [1:7571518518884306702:0] socket: 196 qp: -1
2025-11-11T17:18:50.922108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518519464882317:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:50.922142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:50.922506Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518519464882311:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:50.922790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518515169914937:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:50.922882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518515169914926:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.922926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:50.922970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518515169914926:2050] [node 3] ICP22 created new session: [6:7571518519464882320:2050]
2025-11-11T17:18:50.923000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518519464882320:2050] [node 3] ICS09 handshake done sender: [6:7571518519464882311:2120] self: [6:7571518519464882315:0] peer: [3:7571518520846113258:0] socket: 186 qp: -1
2025-11-11T17:18:50.923027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518519464882320:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:50.923052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:50.697785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518518884306721:2048] [node 2] ICS09 handshake done sender: [1:7571518518884306701:2113] self: [1:7571518518884306700:0] peer: [2:7571518520582467239:0] socket: 193 qp: -1
2025-11-11T17:18:50.697825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518518884306721:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:50.697880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:50.698282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518514589339334:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(39121416ns) {HandleHandshakeStatus:329(39118384ns) {DropIncomingHandshake:441(762ns) DropOutgoingHandshake:456(1839ns) SwitchToState:178(720ns) ProcessPendingSessionEvents:495(92402ns) {ForwardSessionEventToSession:612(49992ns) ForwardSessionEventToSession:612(2680ns) ForwardSessionEventToSession:612(5549ns) ForwardSessionEventToSession:612(2132ns) ForwardSessionEventToSession:612(6924ns) ForwardSessionEventToSession:612(2324ns) ForwardSessionEventToSession:612(2147ns) ForwardSessionEventToSession:612(4840ns) ForwardSessionEventToSession:612(2959ns) ForwardSessionEventToSession:612(2581ns)}}}
2025-11-11T17:18:50.698433Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518518884306703:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:50.699624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518514589339334:2048] [node 2] ICP18 handshake (actor [1:7571518518884306720:2118]) is too old
2025-11-11T17:18:50.749914Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306710:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.749940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518518884306710:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.750495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518518884306695:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.750551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:50.750603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518518884306695:2077] [node 6] ICP22 created new session: [1:7571518518884306727:2077]
2025-11-11T17:18:50.750649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518518884306727:2077] [node 6] ICS09 handshake done sender: [1:7571518518884306703:2114] self: [1:7571518518884306702:0] peer: [6:7571518519464882312:0] socket: 194 qp: -1
2025-11-11T17:18:50.750684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518518884306727:2077] [node 6] ICS10 traffic start
2025-11-11T17:18:50.750739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:50.759487Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518518884306710:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:50.770005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518518884306713:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.770032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518518884306713:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:50.779095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518514589339335:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.780792Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518518884306713:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:50.801838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518518884306694:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.801912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:50.801965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518518884306694:2064] [node 5] ICP22 created new session: [1:7571518518884306731:2064]
2025-11-11T17:18:50.802005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518518884306731:2064] [node 5] ICS09 handshake done sender: [1:7571518518884306713:2117] self: [1:7571518518884306712:0] peer: [5:7571518522124991424:0] socket: 225 qp: -1
2025-11-11T17:18:50.802037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518518884306731:2064] [node 5] ICS10 traffic start
2025-11-11T17:18:50.802068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:50.821722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:50.829684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518514589339335:2049] [node 3] ICP22 created new session: [1:7571518518884306733:2049]
2025-11-11T17:18:50.833809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518518884306733:2049] [node 3] ICS09 handshake done sender: [1:7571518518884306710:2116] self: [1:7571518518884306709:0] peer: [3:7571518520846113272:0] socket: 218 qp: -1
2025-11-11T17:18:50.841762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518518884306733:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:50.841819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:50.842255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518514589339335:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(62960813ns) {HandleHandshakeStatus:329(62957611ns) {DropIncomingHandshake:441(15713ns) DropOutgoingHandshake:456(297ns) SwitchToState:178(4747ns) ProcessPendingSessionEvents:495(101559ns) {ForwardSessionEventToSession:612(57888ns) ForwardSessionEventToSession:612(2918ns) ForwardSessionEventToSession:612(5709ns) ForwardSessionEventToSession:612(4380ns) ForwardSessionEventToSession:612(2724ns) ForwardSessionEventToSession:612(4302ns) ForwardSessionEventToSession:612(2840ns) ForwardSessionEventToSession:612(1891ns) ForwardSessionEventToSession:612(7590ns) ForwardSessionEventToSession:612(2336ns)}}}
2025-11-11T17:18:50.911421Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15326) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.912627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518517830024054:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15326) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518522124991419:2120] held: no
2025-11-11T17:18:50.912706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518517830024054:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.912739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:50.912771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518517830024054:2059] [node 4] ICP22 created new session: [5:7571518522124991434:2059]
2025-11-11T17:18:50.912806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518522124991434:2059] [node 4] ICS09 handshake done sender: [5:7571518522124991419:2120] self: [5:7571518522124991432:0] peer: [4:7571518518396361043:0] socket: 227 qp: -1
2025-11-11T17:18:50.912831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518522124991434:2059] [node 4] ICS10 traffic start
2025-11-11T17:18:50.912859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:50.928367Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518522124991417:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:50.928392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518522124991417:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:50.947113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518519464882322:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54585
2025-11-11T17:18:50.949954Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518522124991417:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:50.950237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518522124991370:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:50.950293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:50.950341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518522124991370:2062] [node 6] ICP22 created new session: [5:7571518522124991437:2062]
2025-11-11T17:18:50.950378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518522124991437:2062] [node 6] ICS09 handshake done sender: [5:7571518522124991417:2118] self: [5:7571518522124991416:0] peer: [6:7571518519464882319:0] socket: 180 qp: -1
2025-11-11T17:18:50.950416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518522124991437:2062] [node 6] ICS10 traffic start
2025-11-11T17:18:50.950445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:50.953909Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:50.955172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518515169914937:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518519464882314:2122] held: no
2025-11-11T17:18:50.955328Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518519464882314:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:50.955573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518515169914937:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:50.955642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:50.955693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518515169914937:2059] [node 5] ICP22 created new session: [6:7571518519464882323:2059]
2025-11-11T17:18:50.955736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518519464882323:2059] [node 5] ICS09 handshake done sender: [6:7571518519464882314:2122] self: [6:7571518519464882319:0] peer: [5:7571518522124991416:0] socket: 189 qp: -1
2025-11-11T17:18:50.955774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518519464882323:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:50.955806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:51.021415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:51.021682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518520582467250:2048] [node 1] ICS01 socket: 201 reason# EndOfStream
2025-11-11T17:18:51.021720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518516287499854:2048] [node 1] ICP30 unregister session Session# [2:7571518520582467250:2048] VirtualId# [2:7571518520582467239:0]
2025-11-11T17:18:51.021757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518520582467250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.021871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:51.021919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:51.025718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:51.025986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518520846113280:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-11T17:18:51.026021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518516551145880:2048] [node 1] ICP30 unregister session Session# [3:7571518520846113280:2048] VirtualId# [3:7571518520846113272:0]
2025-11-11T17:18:51.026053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518520846113280:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.026159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:51.026198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:51.037776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:51.038022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518518396361051:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-11T17:18:51.038055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518514101393669:2048] [node 1] ICP30 unregister session Session# [4:7571518518396361051:2048] VirtualId# [4:7571518518396361048:0]
2025-11-11T17:18:51.038087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518518396361051:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.038155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:51.038190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:51.037776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:51.038021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518522124991430:2048] [node 1] ICS01 socket: 228 reason# EndOfStream
2025-11-11T17:18:51.038056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518517830024038:2048] [node 1] ICP30 unregister session Session# [5:7571518522124991430:2048] VirtualId# [5:7571518522124991424:0]
2025-11-11T17:18:51.038085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518522124991430:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.038153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:51.038193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:51.038295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:51.038492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518519464882317:2048] [node 1] ICS01 socket: 196 reason# EndOfStream
2025-11-11T17:18:51.038520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518515169914924:2048] [node 1] ICP30 unregister session Session# [6:7571518519464882317:2048] VirtualId# [6:7571518519464882312:0]
2025-11-11T17:18:51.038539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518519464882317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.038595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:51.038642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:51.105776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:51.106122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518520846113265:2049] [node 2] ICS01 socket: 188 reason# EndOfStream
2025-11-11T17:18:51.106167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518516551145881:2049] [node 2] ICP30 unregister session Session# [3:7571518520846113265:2049] VirtualId# [3:7571518520846113257:0]
2025-11-11T17:18:51.106196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518520846113265:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.106321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:51.106365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:51.106688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:51.106930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518518396361060:2049] [node 2] ICS01 socket: 175 reason# EndOfStream
2025-11-11T17:18:51.106962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518514101393670:2049] [node 2] ICP30 unregister session Session# [4:7571518518396361060:2049] VirtualId# [4:7571518518396361056:0]
2025-11-11T17:18:51.106987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518518396361060:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.107037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:51.107074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:51.111100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:51.111348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518519464882305:2049] [node 2] ICS01 socket: 190 reason# EndOfStream
2025-11-11T17:18:51.111382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518515169914925:2049] [node 2] ICP30 unregister session Session# [6:7571518519464882305:2049] VirtualId# [6:7571518519464882300:0]
2025-11-11T17:18:51.111415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518519464882305:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.111517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:51.111580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:51.111693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:51.111923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518522124991426:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-11T17:18:51.111962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518517830024039:2049] [node 2] ICP30 unregister session Session# [5:7571518522124991426:2049] VirtualId# [5:7571518522124991413:0]
2025-11-11T17:18:51.111998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518522124991426:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.112063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:51.112115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:51.229803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:51.230017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518519464882320:2050] [node 3] ICS01 socket: 186 reason# EndOfStream
2025-11-11T17:18:51.230060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518515169914926:2050] [node 3] ICP30 unregister session Session# [6:7571518519464882320:2050] VirtualId# [6:7571518519464882315:0]
2025-11-11T17:18:51.230096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518519464882320:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.230174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:51.230214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:51.230288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:51.230446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518522124991425:2050] [node 3] ICS01 socket: 177 reason# EndOfStream
2025-11-11T17:18:51.230474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518517830024040:2050] [node 3] ICP30 unregister session Session# [5:7571518522124991425:2050] VirtualId# [5:7571518522124991414:0]
2025-11-11T17:18:51.230493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518522124991425:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.230546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:51.230575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:51.233707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:51.234006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518518396361067:2050] [node 3] ICS01 socket: 221 reason# EndOfStream
2025-11-11T17:18:51.234050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518514101393671:2050] [node 3] ICP30 unregister session Session# [4:7571518518396361067:2050] VirtualId# [4:7571518518396361058:0]
2025-11-11T17:18:51.234090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518518396361067:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.234206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:51.234273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:51.269796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:51.270029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518522124991434:2059] [node 4] ICS01 socket: 227 reason# EndOfStream
2025-11-11T17:18:51.270070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518517830024054:2059] [node 4] ICP30 unregister session Session# [5:7571518522124991434:2059] VirtualId# [5:7571518522124991432:0]
2025-11-11T17:18:51.270103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518522124991434:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.270221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:51.270269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:51.270369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:51.270551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518519464882308:2064] [node 4] ICS01 socket: 174 reason# EndOfStream
2025-11-11T17:18:51.270581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518519464882284:2064] [node 4] ICP30 unregister session Session# [6:7571518519464882308:2064] VirtualId# [6:7571518519464882296:0]
2025-11-11T17:18:51.270607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518519464882308:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.270674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:51.270713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:51.395712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:51.395987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518519464882323:2059] [node 5] ICS01 socket: 189 reason# EndOfStream
2025-11-11T17:18:51.396031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518515169914937:2059] [node 5] ICP30 unregister session Session# [6:7571518519464882323:2059] VirtualId# [6:7571518519464882319:0]
2025-11-11T17:18:51.396060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518519464882323:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:51.396174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:51.396212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:51.566099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518522591736746:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:51.566133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518522591736747:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:51.646138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518524366670314:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:51.646185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518524366670315:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:51.698441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518526089900985:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:51.698487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518526089900986:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:51.729910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518523796473745:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:51.729956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518523796473746:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:51.729981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518523796473747:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:51.786643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518523502582840:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:51.786694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518523502582841:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:51.786722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518523502582842:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:51.986304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518526127289720:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:51.986342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518526127289721:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:51.986365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518526127289722:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.089688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518528661637644:2072] [node 4] ICP01 ready to work
2025-11-11T17:18:52.105645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518528091441058:2061] [node 6] ICP01 ready to work
2025-11-11T17:18:52.129862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518526886704089:2074] [node 5] ICP01 ready to work
2025-11-11T17:18:52.138053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518530384868316:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:52.154529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518530422257038:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:52.165625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518526886704094:2073] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.185730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518527797550159:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:52.140923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518530384868319:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28020
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.205680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518528661637656:2073] [node 6] ICP01 ready to work
2025-11-11T17:18:52.294970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518527797550159:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(92067718ns) {Configure:104(92060307ns) {ProcessConfigured:135(55764ns) {StartInitialHandshake:157(53253ns) {DropHandshakes:469(6427ns) {DropIncomingHandshake:441(793ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(703ns) PrepareNewSessionHandshake:476(2367ns) {SwitchToState:178(728ns)}}}}}
2025-11-11T17:18:52.185740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518528091441070:2073] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.173647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518530384868325:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:52.173856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518526886704104:2072] [node 6] ICP01 ready to work
2025-11-11T17:18:52.239828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257048:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.305709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518528661637665:2083] [node 5] ICP01 ready to work
2025-11-11T17:18:52.378446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518528661637673:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42604
2025-11-11T17:18:52.378613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518528661637674:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42604
2025-11-11T17:18:52.247174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518522591736747:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(133365302ns) {Configure:104(133357592ns) {ProcessConfigured:135(39280ns) {StartInitialHandshake:157(37089ns) {DropHandshakes:469(1771ns) {DropIncomingHandshake:441(439ns) DropOutgoingHandshake:456(337ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:476(857ns) {SwitchToState:178(463ns)}}}}}
2025-11-11T17:18:52.320262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550193:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.320534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518530422257049:2069] [node 5] ICP01 ready to work
2025-11-11T17:18:52.245192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518530384868325:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19327146ns) {Configure:104(19318330ns) {ProcessConfigured:135(58762ns) {StartInitialHandshake:157(55706ns) {DropHandshakes:469(10283ns) {DropIncomingHandshake:441(685ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(634ns) PrepareNewSessionHandshake:476(1774ns) {SwitchToState:178(506ns)}}}}}
2025-11-11T17:18:52.305615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518530384868345:2064] [node 4] ICP01 ready to work
2025-11-11T17:18:52.326457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518530384868354:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28020
2025-11-11T17:18:52.447445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518530384868345:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(85387435ns) {Configure:104(85378783ns) {ProcessConfigured:135(86693ns) {StartInitialHandshake:157(83392ns) {DropHandshakes:469(3029ns) {DropIncomingHandshake:441(660ns) DropOutgoingHandshake:456(636ns)} GenerateSessionVirtualId:988(1012ns) PrepareNewSessionHandshake:476(2502ns) {SwitchToState:178(959ns)}}}}}
2025-11-11T17:18:52.320665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518527797550169:2066] [node 4] ICP01 ready to work
2025-11-11T17:18:52.248731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518526886704108:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41504
2025-11-11T17:18:52.430730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637673:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.248856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518526886704109:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41504
2025-11-11T17:18:52.418241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550208:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
2025-11-11T17:18:52.248908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518526886704110:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41504
2025-11-11T17:18:52.321652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704108:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.394229Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704110:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.501643Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704114:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.501685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518526886704114:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.466382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518524366670314:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(376399604ns) {Configure:104(376394460ns) {ProcessConfigured:135(69181ns) {StartInitialHandshake:157(64078ns) {DropHandshakes:469(3193ns) {DropIncomingHandshake:441(743ns) DropOutgoingHandshake:456(604ns)} GenerateSessionVirtualId:988(1423ns) PrepareNewSessionHandshake:476(1922ns) {SwitchToState:178(625ns)}}}}}
2025-11-11T17:18:52.477883Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868354:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.426771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518523502582842:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(269010740ns) {Configure:104(269005502ns) {ProcessConfigured:135(77872ns) {StartInitialHandshake:157(75304ns) {DropHandshakes:469(3038ns) {DropIncomingHandshake:441(640ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(895ns) PrepareNewSessionHandshake:476(2630ns) {SwitchToState:178(925ns)}}}}}
2025-11-11T17:18:52.493855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518526089900985:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.521852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518526886704104:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(239097244ns) {Configure:104(239090733ns) {ProcessConfigured:135(56668ns) {StartInitialHandshake:157(54535ns) {DropHandshakes:469(3775ns) {DropIncomingHandshake:441(1270ns) DropOutgoingHandshake:456(706ns)} GenerateSessionVirtualId:988(743ns) PrepareNewSessionHandshake:476(1732ns) {SwitchToState:178(564ns)}}}}}
2025-11-11T17:18:52.474759Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637674:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.473785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518527797550193:2097] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.533812Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637678:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.533851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518528661637678:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.538119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518524366670315:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(432202818ns) {Configure:104(432195311ns) {ProcessConfigured:135(4279248ns) {StartInitialHandshake:157(4275964ns) {DropHandshakes:469(3850ns) {DropIncomingHandshake:441(950ns) DropOutgoingHandshake:456(569ns)} GenerateSessionVirtualId:988(756ns) PrepareNewSessionHandshake:476(2776ns) {SwitchToState:178(1147ns)}}}}}
2025-11-11T17:18:52.550645Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518528661637678:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:52.558353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518523796473747:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(455076738ns) {Configure:104(455071156ns) {ProcessConfigured:135(50579ns) {StartInitialHandshake:157(48767ns) {DropHandshakes:469(545ns) {DropIncomingHandshake:441(120ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(126ns) PrepareNewSessionHandshake:476(2469ns) {SwitchToState:178(813ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.541757Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868319:2082] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.561727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868321:2083] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.561765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518530384868321:2083] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.562221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518530384868364:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28020
2025-11-11T17:18:52.565868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518528661637644:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(439375940ns) {Configure:104(439367496ns) {ProcessConfigured:135(27922837ns) {StartInitialHandshake:157(27919286ns) {DropHandshakes:469(1780ns) {DropIncomingHandshake:441(428ns) DropOutgoingHandshake:456(280ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:476(3301ns) {SwitchToState:178(1605ns)}}}}}
2025-11-11T17:18:52.505931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518530422257048:2074] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.533809Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704112:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.533846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518526886704112:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.541777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518522591736746:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(371896620ns) {Configure:104(371891734ns) {ProcessConfigured:135(24054701ns) {StartInitialHandshake:157(24052414ns) {DropHandshakes:469(2619ns) {DropIncomingHandshake:441(298ns) DropOutgoingHandshake:456(637ns)} GenerateSessionVirtualId:988(818ns) PrepareNewSessionHandshake:476(2604ns) {SwitchToState:178(919ns)}}}}}
2025-11-11T17:18:52.545796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518526886704123:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41504
2025-11-11T17:18:52.557914Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518526886704114:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:52.473947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518523502582840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.585955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518528091441058:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(359921135ns) {Configure:104(359916661ns) {ProcessConfigured:135(7998755ns) {StartInitialHandshake:157(7994495ns) {DropHandshakes:469(2254ns) {DropIncomingHandshake:441(502ns) DropOutgoingHandshake:456(496ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:476(3585ns) {SwitchToState:178(1409ns)}}}}}
2025-11-11T17:18:52.506356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257088:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
2025-11-11T17:18:52.486962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518527797550169:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(105018381ns) {Configure:104(105013251ns) {ProcessConfigured:135(85583ns) {StartInitialHandshake:157(82238ns) {DropHandshakes:469(3872ns) {DropIncomingHandshake:441(546ns) DropOutgoingHandshake:456(493ns)} GenerateSessionVirtualId:988(1480ns) PrepareNewSessionHandshake:476(2778ns) {SwitchToState:178(777ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.578725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518526886704112:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:52.513784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518527797550208:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.506515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257089:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.598908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518530384868363:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28020
2025-11-11T17:18:52.579079Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704109:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.579201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518526886704089:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.579260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:52.579306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518526886704089:2074] [node 5] ICP22 created new session: [1:7571518526886704128:2074]
2025-11-11T17:18:52.579350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518526886704128:2074] [node 5] ICS09 handshake done sender: [1:7571518526886704114:2114] self: [1:7571518526886704113:0] peer: [5:7571518527797550214:0] socket: 186 qp: -1
2025-11-11T17:18:52.579384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518526886704128:2074] [node 5] ICS10 traffic start
2025-11-11T17:18:52.579415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:52.594454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518526886704094:2073] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(355406926ns) {Configure:104(355401522ns) {ProcessConfigured:135(19564510ns) {StartInitialHandshake:157(19562826ns) {DropHandshakes:469(3200ns) {DropIncomingHandshake:441(719ns) DropOutgoingHandshake:456(684ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:476(2702ns) {SwitchToState:178(973ns)}}}}}
2025-11-11T17:18:52.610699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518522591736747:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.610793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:52.610840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518522591736747:2049] [node 3] ICP22 created new session: [1:7571518526886704133:2049]
2025-11-11T17:18:52.610885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518526886704133:2049] [node 3] ICS09 handshake done sender: [1:7571518526886704112:2113] self: [1:7571518526886704111:0] peer: [3:7571518530384868362:0] socket: 185 qp: -1
2025-11-11T17:18:52.610920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518526886704133:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:52.610966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:52.623711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704123:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.507837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518526127289722:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.612058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637676:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.514149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518523502582841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.599384Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518530384868321:2083] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.612097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518528661637676:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.522583Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518530422257088:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.674335Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518530384868354:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.677966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441119:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.678494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441120:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.678702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441121:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.673726Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704121:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.673780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518526886704121:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.539720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550218:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.685643Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704122:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.685697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518526886704122:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.522661Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518530422257089:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.560803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550219:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
2025-11-11T17:18:52.612806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518528661637665:2083] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.679167Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.697894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518526886704121:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:52.681631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518523796473746:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(474849806ns) {Configure:104(474846742ns) {ProcessConfigured:135(32871634ns) {StartInitialHandshake:157(32868282ns) {DropHandshakes:469(707ns) {DropIncomingHandshake:441(174ns) DropOutgoingHandshake:456(247ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(1779ns) {SwitchToState:178(628ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.561399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550220:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
2025-11-11T17:18:52.617832Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.622051Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518527797550193:2097] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:52.698184Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518526089900985:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518530384868354:2112] held: no
2025-11-11T17:18:52.624305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518523502582840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518527797550193:2097] held: no
2025-11-11T17:18:52.612872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:52.681833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518523796473745:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(518193460ns) {Configure:104(518184720ns) {ProcessConfigured:135(38003901ns) {StartInitialHandshake:157(38001698ns) {DropHandshakes:469(3282ns) {DropIncomingHandshake:441(731ns) DropOutgoingHandshake:456(590ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:476(2436ns) {SwitchToState:178(806ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.624377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518523502582840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.699455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518526886704104:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.529876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518530422257038:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(359920621ns) {Configure:104(359915911ns) {ProcessConfigured:135(7781168ns) {StartInitialHandshake:157(7777898ns) {DropHandshakes:469(3132ns) {DropIncomingHandshake:441(673ns) DropOutgoingHandshake:456(690ns)} GenerateSessionVirtualId:988(625ns) PrepareNewSessionHandshake:476(3946ns) {SwitchToState:178(1573ns)}}}}}
2025-11-11T17:18:52.612916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518528661637665:2083] [node 5] ICP22 created new session: [2:7571518528661637695:2083]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.624439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:52.698464Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868364:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.561184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257094:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.701781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:52.701654Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518528091441120:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.698525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518530384868316:2066] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.612964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518528661637695:2083] [node 5] ICS09 handshake done sender: [2:7571518528661637678:2110] self: [2:7571518528661637677:0] peer: [5:7571518527797550217:0] socket: 194 qp: -1
2025-11-11T17:18:52.706062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441122:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.715403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518523796473745:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.717989Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518528091441110:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.718026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518528091441110:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.734063Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518528091441110:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:52.806274Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518528091441122:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.806557Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518528091441119:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.807053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441125:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.807202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441126:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.807515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518528091441070:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.807587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:52.807635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518528091441070:2073] [node 5] ICP22 created new session: [4:7571518528091441129:2073]
2025-11-11T17:18:52.807677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518528091441129:2073] [node 5] ICS09 handshake done sender: [4:7571518528091441110:2111] self: [4:7571518528091441109:0] peer: [5:7571518527797550227:0] socket: 209 qp: -1
2025-11-11T17:18:52.807712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518528091441129:2073] [node 5] ICS10 traffic start
2025-11-11T17:18:52.807761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:52.809225Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518528091441120:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:52.809734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518523796473745:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.809805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:52.809845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518523796473745:2048] [node 1] ICP22 created new session: [4:7571518528091441132:2048]
2025-11-11T17:18:52.809891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518528091441132:2048] [node 1] ICS09 handshake done sender: [4:7571518528091441120:2116] self: [4:7571518528091441123:0] peer: [1:7571518526886704127:0] socket: 219 qp: -1
2025-11-11T17:18:52.809918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518528091441132:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:52.809946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:52.624484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518523502582840:2048] [node 1] ICP22 created new session: [5:7571518527797550221:2048]
2025-11-11T17:18:52.701818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518526886704104:2072] [node 6] ICP22 created new session: [1:7571518526886704139:2072]
2025-11-11T17:18:52.569877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518526127289721:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(395823353ns) {Configure:104(395816626ns) {ProcessConfigured:135(11818711ns) {StartInitialHandshake:157(11814805ns) {DropHandshakes:469(3379ns) {DropIncomingHandshake:441(784ns) DropOutgoingHandshake:456(678ns)} GenerateSessionVirtualId:988(660ns) PrepareNewSessionHandshake:476(3227ns) {SwitchToState:178(1224ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.833653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518528091441125:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.612990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518528661637695:2083] [node 5] ICS10 traffic start
2025-11-11T17:18:52.624518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518527797550221:2048] [node 1] ICS09 handshake done sender: [5:7571518527797550193:2097] self: [5:7571518527797550214:0] peer: [1:7571518526886704113:0] socket: 188 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.698583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:52.713938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518526886704139:2072] [node 6] ICS09 handshake done sender: [1:7571518526886704121:2115] self: [1:7571518526886704120:0] peer: [6:7571518530422257098:0] socket: 204 qp: -1
2025-11-11T17:18:52.570079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518526127289721:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.698623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518530384868316:2066] [node 6] ICP22 created new session: [3:7571518530384868368:2066]
2025-11-11T17:18:52.833838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518523796473747:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.613024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:52.613263Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518530422257094:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.624548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518527797550221:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:52.713985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518526886704139:2072] [node 6] ICS10 traffic start
2025-11-11T17:18:52.623187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518528661637694:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42604
2025-11-11T17:18:52.698664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518530384868368:2066] [node 6] ICS09 handshake done sender: [3:7571518530384868321:2083] self: [3:7571518530384868320:0] peer: [6:7571518530422257090:0] socket: 179 qp: -1
2025-11-11T17:18:52.714036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:52.717691Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518526886704122:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:52.749784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518526886704132:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.749809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518526886704132:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.762137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518522591736746:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.762218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:52.762269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518522591736746:2048] [node 2] ICP22 created new session: [1:7571518526886704141:2048]
2025-11-11T17:18:52.762311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518526886704141:2048] [node 2] ICS09 handshake done sender: [1:7571518526886704122:2116] self: [1:7571518526886704119:0] peer: [2:7571518528661637702:0] socket: 214 qp: -1
2025-11-11T17:18:52.762348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518526886704141:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:52.762399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:52.785840Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518526886704132:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:52.849734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518526886704094:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.849782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:52.865976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518526886704094:2073] [node 4] ICP22 created new session: [1:7571518526886704144:2073]
2025-11-11T17:18:52.897929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518526886704144:2073] [node 4] ICS09 handshake done sender: [1:7571518526886704132:2118] self: [1:7571518526886704127:0] peer: [4:7571518528091441123:0] socket: 217 qp: -1
2025-11-11T17:18:52.905692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518526886704144:2073] [node 4] ICS10 traffic start
2025-11-11T17:18:52.917696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:52.622924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518526127289720:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.624579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:52.624978Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518527797550208:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:52.625368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518523502582841:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.625422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:52.625459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518523502582841:2049] [node 2] ICP22 created new session: [5:7571518527797550223:2049]
2025-11-11T17:18:52.625494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518527797550223:2049] [node 2] ICS09 handshake done sender: [5:7571518527797550208:2111] self: [5:7571518527797550217:0] peer: [2:7571518528661637677:0] socket: 195 qp: -1
2025-11-11T17:18:52.625519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518527797550223:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:52.625541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:52.701520Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518527797550220:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.713813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518527797550169:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.722399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550228:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
2025-11-11T17:18:52.722447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550229:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
2025-11-11T17:18:52.750082Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518527797550220:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:52.773666Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518527797550228:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.774550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518527797550169:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.774622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:52.774692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518527797550169:2066] [node 4] ICP22 created new session: [5:7571518527797550230:2066]
2025-11-11T17:18:52.774733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518527797550230:2066] [node 4] ICS09 handshake done sender: [5:7571518527797550220:2119] self: [5:7571518527797550227:0] peer: [4:7571518528091441109:0] socket: 210 qp: -1
2025-11-11T17:18:52.774767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518527797550230:2066] [node 4] ICS10 traffic start
2025-11-11T17:18:52.774801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:52.778166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518523502582842:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.813821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518527797550177:2084] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.813855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518527797550177:2084] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.815037Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518527797550177:2084] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:52.815283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518527797550159:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.815327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:52.815371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518527797550159:2064] [node 6] ICP22 created new session: [5:7571518527797550234:2064]
2025-11-11T17:18:52.815416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518527797550234:2064] [node 6] ICS09 handshake done sender: [5:7571518527797550177:2084] self: [5:7571518527797550176:0] peer: [6:7571518530422257105:0] socket: 189 qp: -1
2025-11-11T17:18:52.815453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518527797550234:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:52.815490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:52.845055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518527797550236:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24931
2025-11-11T17:18:52.918191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518527797550228:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:52.925788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518523502582842:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.945814Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.858015Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518523796473745:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(48068688ns) {HandleHandshakeStatus:329(48066620ns) {DropIncomingHandshake:441(572ns) DropOutgoingHandshake:456(3998ns) SwitchToState:178(297ns) ProcessPendingSessionEvents:495(16006583ns) {ForwardSessionEventToSession:612(15924782ns) ForwardSessionEventToSession:612(29391ns) ForwardSessionEventToSession:612(3192ns) ForwardSessionEventToSession:612(5125ns) ForwardSessionEventToSession:612(2198ns) ForwardSessionEventToSession:612(3365ns) ForwardSessionEventToSession:612(5324ns) ForwardSessionEventToSession:612(3360ns) ForwardSessionEventToSession:612(8595ns) ForwardSessionEventToSession:612(7359ns)}}}
2025-11-11T17:18:52.961700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:52.662021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257099:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
2025-11-11T17:18:52.638103Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518528661637676:2109] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:52.654971Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637694:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.655021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518528661637656:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.655070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:52.655115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518528661637656:2073] [node 6] ICP22 created new session: [2:7571518528661637699:2073]
2025-11-11T17:18:52.655159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518528661637699:2073] [node 6] ICS09 handshake done sender: [2:7571518528661637676:2109] self: [2:7571518528661637675:0] peer: [6:7571518530422257097:0] socket: 193 qp: -1
2025-11-11T17:18:52.655193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518528661637699:2073] [node 6] ICS10 traffic start
2025-11-11T17:18:52.655235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:52.656002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518524366670314:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.776959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637691:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.776981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518528661637691:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.786657Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.787516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518524366670314:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518528661637694:2117] held: no
2025-11-11T17:18:52.789896Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518528661637691:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:52.798660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518528661637703:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42604
2025-11-11T17:18:52.833985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518524366670315:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.834045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:52.834086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518524366670315:2049] [node 3] ICP22 created new session: [2:7571518528661637705:2049]
2025-11-11T17:18:52.834126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518528661637705:2049] [node 3] ICS09 handshake done sender: [2:7571518528661637691:2115] self: [2:7571518528661637688:0] peer: [3:7571518530384868377:0] socket: 218 qp: -1
2025-11-11T17:18:52.834166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518528661637705:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:52.834195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:52.842017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518524366670314:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16269017ns)
2025-11-11T17:18:52.842541Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518528661637694:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:52.867298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518524366670314:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.867348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:52.867392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518524366670314:2048] [node 1] ICP22 created new session: [2:7571518528661637708:2048]
2025-11-11T17:18:52.867428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518528661637708:2048] [node 1] ICS09 handshake done sender: [2:7571518528661637694:2117] self: [2:7571518528661637702:0] peer: [1:7571518526886704119:0] socket: 182 qp: -1
2025-11-11T17:18:52.867466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518528661637708:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:52.867498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:52.868009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518528661637707:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42604
2025-11-11T17:18:52.909501Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637693:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.909532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518528661637693:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.918009Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518528661637707:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.922801Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518528661637693:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:52.923073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518528661637644:2072] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.923142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:52.923186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518528661637644:2072] [node 4] ICP22 created new session: [2:7571518528661637712:2072]
2025-11-11T17:18:52.923227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518528661637712:2072] [node 4] ICS09 handshake done sender: [2:7571518528661637693:2116] self: [2:7571518528661637690:0] peer: [4:7571518528091441139:0] socket: 222 qp: -1
2025-11-11T17:18:52.923261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518528661637712:2072] [node 4] ICS10 traffic start
2025-11-11T17:18:52.923290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:52.698699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518530384868368:2066] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:52.869859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518528091441114:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.869885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518528091441114:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.871510Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518528091441114:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:52.877904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441135:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.886218Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518528091441121:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.886275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518528091441058:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.886330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:52.666091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257101:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
2025-11-11T17:18:52.713815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257100:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
2025-11-11T17:18:52.745911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518530422257049:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(312935363ns) {Configure:104(296290185ns) {ProcessConfigured:135(19888599ns) {StartInitialHandshake:157(19885928ns) {DropHandshakes:469(8014ns) {DropIncomingHandshake:441(769ns) DropOutgoingHandshake:456(570ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:476(3054ns) {SwitchToState:178(1092ns)}}}}}
2025-11-11T17:18:52.746122Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.775896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518530422257049:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.776069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518526127289722:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518530422257048:2074] held: no
2025-11-11T17:18:52.776205Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.786238Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518530422257048:2074] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:52.786454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518526127289720:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518530422257094:2116] held: no
2025-11-11T17:18:52.786556Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518530422257089:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:52.798286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257104:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
2025-11-11T17:18:52.817890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518526127289722:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.817949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:52.817991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518526127289722:2050] [node 3] ICP22 created new session: [6:7571518530422257106:2050]
2025-11-11T17:18:52.818033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518530422257106:2050] [node 3] ICS09 handshake done sender: [6:7571518530422257048:2074] self: [6:7571518530422257090:0] peer: [3:7571518530384868320:0] socket: 181 qp: -1
2025-11-11T17:18:52.818065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518530422257106:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:52.818108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:52.818529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518526127289721:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.818595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:52.818635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518526127289721:2049] [node 2] ICP22 created new session: [6:7571518530422257108:2049]
2025-11-11T17:18:52.818703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518530422257108:2049] [node 2] ICS09 handshake done sender: [6:7571518530422257089:2114] self: [6:7571518530422257097:0] peer: [2:7571518528661637675:0] socket: 197 qp: -1
2025-11-11T17:18:52.818734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518530422257108:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:52.818779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:52.819379Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518530422257099:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.819945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518530422257094:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:52.820788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257110:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
2025-11-11T17:18:52.821193Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518530422257088:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:52.821374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518530422257049:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.821412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:52.821468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518530422257049:2069] [node 5] ICP22 created new session: [6:7571518530422257111:2069]
2025-11-11T17:18:52.821503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518530422257111:2069] [node 5] ICS09 handshake done sender: [6:7571518530422257088:2113] self: [6:7571518530422257105:0] peer: [5:7571518527797550176:0] socket: 196 qp: -1
2025-11-11T17:18:52.821533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518530422257111:2069] [node 5] ICS10 traffic start
2025-11-11T17:18:52.821576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:52.861838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518526127289720:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.861919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:52.861970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518526127289720:2048] [node 1] ICP22 created new session: [6:7571518530422257114:2048]
2025-11-11T17:18:52.862021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518530422257114:2048] [node 1] ICS09 handshake done sender: [6:7571518530422257094:2116] self: [6:7571518530422257098:0] peer: [1:7571518526886704120:0] socket: 207 qp: -1
2025-11-11T17:18:52.862056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518530422257114:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:52.862119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:52.863021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518530422257038:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.866315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518530422257049:2069] [node 5] ICP35 event processing took too much time PendingConnection:278(44700069ns) {HandleHandshakeStatus:329(44698469ns) {DropIncomingHandshake:441(340ns) DropOutgoingHandshake:456(3484ns) SwitchToState:178(228ns) ProcessPendingSessionEvents:495(32402178ns) {ForwardSessionEventToSession:612(12150777ns) ForwardSessionEventToSession:612(7037ns) ForwardSessionEventToSession:612(6106ns) ForwardSessionEventToSession:612(2208ns) ForwardSessionEventToSession:612(5305ns) ForwardSessionEventToSession:612(5958ns) ForwardSessionEventToSession:612(1925ns) ForwardSessionEventToSession:612(4373ns) ForwardSessionEventToSession:612(20195142ns) ForwardSessionEventToSession:612(9567ns)}}}
2025-11-11T17:18:52.871282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518530422257118:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31510
2025-11-11T17:18:52.872050Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518530422257099:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:52.872282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518530422257038:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.872355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:52.872400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518530422257038:2064] [node 4] ICP22 created new session: [6:7571518530422257119:2064]
2025-11-11T17:18:52.872444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518530422257119:2064] [node 4] ICS09 handshake done sender: [6:7571518530422257099:2118] self: [6:7571518530422257117:0] peer: [4:7571518528091441113:0] socket: 215 qp: -1
2025-11-11T17:18:52.872479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518530422257119:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:52.872517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:52.886386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518528091441058:2061] [node 6] ICP22 created new session: [4:7571518528091441137:2061]
2025-11-11T17:18:52.698730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:52.699657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518530384868367:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28020
2025-11-11T17:18:52.699863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518526089900985:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.699893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:52.699925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518526089900985:2048] [node 1] ICP22 created new session: [3:7571518530384868370:2048]
2025-11-11T17:18:52.699970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518530384868370:2048] [node 1] ICS09 handshake done sender: [3:7571518530384868354:2112] self: [3:7571518530384868362:0] peer: [1:7571518526886704111:0] socket: 187 qp: -1
2025-11-11T17:18:52.700003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518530384868370:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:52.700027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:52.718062Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868367:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.757999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518530384868374:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28020
2025-11-11T17:18:52.763163Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868374:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.795943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518526089900986:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.801769Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868347:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.813726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518530384868347:2104] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.820187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518530384868378:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28020
2025-11-11T17:18:52.829797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518526089900986:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(54051744ns) {IncomingHandshake:261(30296356ns) {DropIncomingHandshake:441(414ns) PrepareNewSessionHandshake:476(1276ns) {SwitchToState:178(726ns)} GenerateSessionVirtualId:988(513ns) IssueIncomingHandshakeReply:193(30266354ns) {SwitchToState:178(2903ns)}}}
2025-11-11T17:18:52.843468Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518530384868361:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:52.843492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518530384868361:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:52.845906Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518530384868347:2104] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:52.865697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518530384868361:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:52.869674Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518530384868367:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:52.905771Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27814) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.906434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518530384868325:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.906536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:52.906593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518530384868325:2062] [node 5] ICP22 created new session: [3:7571518530384868381:2062]
2025-11-11T17:18:52.906635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518530384868381:2062] [node 5] ICS09 handshake done sender: [3:7571518530384868347:2104] self: [3:7571518530384868346:0] peer: [5:7571518527797550232:0] socket: 228 qp: -1
2025-11-11T17:18:52.906669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518530384868381:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:52.906703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:52.907112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518530384868345:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:52.907151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:52.907191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518530384868345:2064] [node 4] ICP22 created new session: [3:7571518530384868383:2064]
2025-11-11T17:18:52.907233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518530384868383:2064] [node 4] ICS09 handshake done sender: [3:7571518530384868361:2116] self: [3:7571518530384868360:0] peer: [4:7571518528091441134:0] socket: 176 qp: -1
2025-11-11T17:18:52.907258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518530384868383:2064] [node 4] ICS10 traffic start
2025-11-11T17:18:52.907284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:52.907588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518526089900986:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.907654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:52.907690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518526089900986:2049] [node 2] ICP22 created new session: [3:7571518530384868385:2049]
2025-11-11T17:18:52.907744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518530384868385:2049] [node 2] ICS09 handshake done sender: [3:7571518530384868367:2119] self: [3:7571518530384868377:0] peer: [2:7571518528661637688:0] socket: 174 qp: -1
2025-11-11T17:18:52.907790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518530384868385:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:52.907813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:52.908012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518526089900986:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:52.997810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518526886704094:2073] [node 4] ICP35 event processing took too much time PendingConnection:278(147712702ns) {HandleHandshakeStatus:329(139964361ns) {DropIncomingHandshake:441(582ns) DropOutgoingHandshake:456(1279ns) SwitchToState:178(684ns) ProcessPendingSessionEvents:495(47927491ns) {ForwardSessionEventToSession:612(27932274ns) ForwardSessionEventToSession:612(4853469ns) ForwardSessionEventToSession:612(7300ns) ForwardSessionEventToSession:612(2355ns) ForwardSessionEventToSession:612(4186624ns) ForwardSessionEventToSession:612(3664ns) ForwardSessionEventToSession:612(2511ns) ForwardSessionEventToSession:612(3659858ns) ForwardSessionEventToSession:612(7234904ns) ForwardSessionEventToSession:612(13374ns)}}}
2025-11-11T17:18:52.981669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518523502582842:2050] [node 3] ICP22 created new session: [5:7571518527797550237:2050]
2025-11-11T17:18:52.981748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518527797550237:2050] [node 3] ICS09 handshake done sender: [5:7571518527797550228:2120] self: [5:7571518527797550232:0] peer: [3:7571518530384868346:0] socket: 231 qp: -1
2025-11-11T17:18:52.981791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518527797550237:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:52.981844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:52.886431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518528091441137:2061] [node 6] ICS09 handshake done sender: [4:7571518528091441114:2112] self: [4:7571518528091441113:0] peer: [6:7571518530422257117:0] socket: 213 qp: -1
2025-11-11T17:18:52.886465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518528091441137:2061] [node 6] ICS10 traffic start
2025-11-11T17:18:52.886503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:52.887565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518523796473746:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:52.898187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518528091441125:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:52.918491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518523796473747:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:52.918580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:52.918626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518523796473747:2050] [node 3] ICP22 created new session: [4:7571518528091441140:2050]
2025-11-11T17:18:52.918700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518528091441140:2050] [node 3] ICS09 handshake done sender: [4:7571518528091441125:2119] self: [4:7571518528091441134:0] peer: [3:7571518530384868360:0] socket: 175 qp: -1
2025-11-11T17:18:52.918735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518528091441140:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:52.918787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:52.934073Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.953702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518523796473747:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:52.955132Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27814) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:52.974073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518523796473746:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27814) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518528091441121:2117] held: no
2025-11-11T17:18:52.982076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518528091441142:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8505
2025-11-11T17:18:52.995005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518528091441121:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:53.001675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518523796473746:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:53.001778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:53.001829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518523796473746:2049] [node 2] ICP22 created new session: [4:7571518532386408439:2049]
2025-11-11T17:18:53.001880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518532386408439:2049] [node 2] ICS09 handshake done sender: [4:7571518528091441121:2117] self: [4:7571518528091441139:0] peer: [2:7571518528661637690:0] socket: 223 qp: -1
2025-11-11T17:18:53.001912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518532386408439:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:53.001970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:53.002964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518523502582842:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(76966597ns) {HandleHandshakeStatus:329(76964276ns) {DropIncomingHandshake:441(694ns) DropOutgoingHandshake:456(3835ns) SwitchToState:178(232ns) ProcessPendingSessionEvents:495(3836398ns) {ForwardSessionEventToSession:612(3785961ns) ForwardSessionEventToSession:612(5415ns) ForwardSessionEventToSession:612(9502ns) ForwardSessionEventToSession:612(4577ns) ForwardSessionEventToSession:612(2788ns) ForwardSessionEventToSession:612(2908ns) ForwardSessionEventToSession:612(5182ns) ForwardSessionEventToSession:612(2929ns) ForwardSessionEventToSession:612(4353ns) ForwardSessionEventToSession:612(1694ns)}}}
2025-11-11T17:18:53.003030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518523502582842:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:53.071189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:53.071436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518528661637708:2048] [node 1] ICS01 socket: 182 reason# EndOfStream
2025-11-11T17:18:53.071474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518524366670314:2048] [node 1] ICP30 unregister session Session# [2:7571518528661637708:2048] VirtualId# [2:7571518528661637702:0]
2025-11-11T17:18:53.071503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518528661637708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.071576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:53.071618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:53.083187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:53.083423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518530384868370:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-11T17:18:53.083457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518526089900985:2048] [node 1] ICP30 unregister session Session# [3:7571518530384868370:2048] VirtualId# [3:7571518530384868362:0]
2025-11-11T17:18:53.083489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518530384868370:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.083568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:53.083608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:53.101114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:53.101358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518528091441132:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-11T17:18:53.101428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518523796473745:2048] [node 1] ICP30 unregister session Session# [4:7571518528091441132:2048] VirtualId# [4:7571518528091441123:0]
2025-11-11T17:18:53.101769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:53.101466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518528091441132:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.101949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518527797550221:2048] [node 1] ICS01 socket: 188 reason# EndOfStream
2025-11-11T17:18:53.101559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:53.101977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518523502582840:2048] [node 1] ICP30 unregister session Session# [5:7571518527797550221:2048] VirtualId# [5:7571518527797550214:0]
2025-11-11T17:18:53.101618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:53.102014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518527797550221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.102074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:53.102110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:53.101109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:53.102548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518530422257114:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-11-11T17:18:53.102592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518526127289720:2048] [node 1] ICP30 unregister session Session# [6:7571518530422257114:2048] VirtualId# [6:7571518530422257098:0]
2025-11-11T17:18:53.102613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518530422257114:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.102666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:53.102701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:53.151344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:53.151563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518530384868385:2049] [node 2] ICS01 socket: 174 reason# EndOfStream
2025-11-11T17:18:53.151600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518526089900986:2049] [node 2] ICP30 unregister session Session# [3:7571518530384868385:2049] VirtualId# [3:7571518530384868377:0]
2025-11-11T17:18:53.151630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518530384868385:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.151707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:53.151747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:53.162545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:53.162771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518532386408439:2049] [node 2] ICS01 socket: 223 reason# EndOfStream
2025-11-11T17:18:53.162807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518523796473746:2049] [node 2] ICP30 unregister session Session# [4:7571518532386408439:2049] VirtualId# [4:7571518528091441139:0]
2025-11-11T17:18:53.162837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518532386408439:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.162914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:53.162959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:53.165708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:53.165918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518530422257108:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-11T17:18:53.165952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518526127289721:2049] [node 2] ICP30 unregister session Session# [6:7571518530422257108:2049] VirtualId# [6:7571518530422257097:0]
2025-11-11T17:18:53.165980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518530422257108:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.166060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:53.166097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:53.166570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:53.166725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518527797550223:2049] [node 2] ICS07 socket disconnect 195 reason# EndOfStream
2025-11-11T17:18:53.166752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518527797550223:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.166823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:53.166873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518527797550223:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:53.174113Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27814) Couldn't connect to any resolved address
2025-11-11T17:18:53.174375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518523502582841:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27814) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:53.174424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518527797550223:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:53.174457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518523502582841:2049] [node 2] ICP30 unregister session Session# [5:7571518527797550223:2049] VirtualId# [5:7571518527797550217:0]
2025-11-11T17:18:53.174502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:53.174535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518523502582841:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27814) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:53.174568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27814) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:53.185731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518523502582841:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:53.267822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:53.268067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518528091441140:2050] [node 3] ICS01 socket: 175 reason# EndOfStream
2025-11-11T17:18:53.268103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518523796473747:2050] [node 3] ICP30 unregister session Session# [4:7571518528091441140:2050] VirtualId# [4:7571518528091441134:0]
2025-11-11T17:18:53.268156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518528091441140:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.268243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:53.268300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:53.268819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:53.269008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518530422257106:2050] [node 3] ICS01 socket: 181 reason# EndOfStream
2025-11-11T17:18:53.269034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518526127289722:2050] [node 3] ICP30 unregister session Session# [6:7571518530422257106:2050] VirtualId# [6:7571518530422257090:0]
2025-11-11T17:18:53.269059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518530422257106:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.269117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:53.269147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:53.268738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:53.273793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518527797550237:2050] [node 3] ICS01 socket: 231 reason# EndOfStream
2025-11-11T17:18:53.273826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518523502582842:2050] [node 3] ICP30 unregister session Session# [5:7571518527797550237:2050] VirtualId# [5:7571518527797550232:0]
2025-11-11T17:18:53.273859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518527797550237:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.273937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:53.273972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:53.413783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:53.414015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518527797550230:2066] [node 4] ICS01 socket: 210 reason# EndOfStream
2025-11-11T17:18:53.414050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518527797550169:2066] [node 4] ICP30 unregister session Session# [5:7571518527797550230:2066] VirtualId# [5:7571518527797550227:0]
2025-11-11T17:18:53.414079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518527797550230:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.414162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:53.414205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:53.414292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:53.414464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518530422257119:2064] [node 4] ICS01 socket: 215 reason# EndOfStream
2025-11-11T17:18:53.414492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518530422257038:2064] [node 4] ICP30 unregister session Session# [6:7571518530422257119:2064] VirtualId# [6:7571518530422257117:0]
2025-11-11T17:18:53.414514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518530422257119:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.414562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:53.414597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:53.521754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:53.521980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518530422257111:2069] [node 5] ICS07 socket disconnect 196 reason# EndOfStream
2025-11-11T17:18:53.522008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518530422257111:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:53.522120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:53.522164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518530422257111:2069] [node 5] ICS15 start handshake
2025-11-11T17:18:53.522820Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25441) Couldn't connect to any resolved address
2025-11-11T17:18:53.523031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518530422257049:2069] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25441) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:53.523083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518530422257111:2069] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:53.523115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518530422257049:2069] [node 5] ICP30 unregister session Session# [6:7571518530422257111:2069] VirtualId# [6:7571518530422257105:0]
2025-11-11T17:18:53.523159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:18:53.523185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518530422257049:2069] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25441) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:53.523200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25441) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:53.533707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518530422257049:2069] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:53.872352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518532609716287:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:53.872401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518532609716288:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:53.933929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518534621745886:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:53.933968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518534621745887:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:53.981870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518532848953808:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:53.981908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518532848953809:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:54.053908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518536996007061:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:54.053953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518536996007062:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:54.053975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518536996007063:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:54.125897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518535907591526:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:54.125938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518535907591527:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:54.125962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518535907591528:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:54.369849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518539462928123:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:54.369918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518539462928124:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:54.369945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518539462928125:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:54.370516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518539462928139:2060] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.406029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518536996007078:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
2025-11-11T17:18:54.328731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518538916713199:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.423250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518536904683629:2071] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.425661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518536904683621:2069] [node 5] ICP01 ready to work
2025-11-11T17:18:54.423301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518537143921129:2059] [node 5] ICP01 ready to work
2025-11-11T17:18:54.507853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518538916713226:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25155
2025-11-11T17:18:54.446317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518537143921137:2076] [node 6] ICP01 ready to work
2025-11-11T17:18:54.497837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518536996007079:2063] [node 6] ICP01 ready to work
2025-11-11T17:18:54.601844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518536996007124:2071] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.616605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518539462928187:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:54.425824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518536904683622:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.667949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518532609716287:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(269727808ns) {Configure:104(269720726ns) {ProcessConfigured:135(104875ns) {StartInitialHandshake:157(102368ns) {DropHandshakes:469(3133ns) {DropIncomingHandshake:441(644ns) DropOutgoingHandshake:456(574ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:476(2462ns) {SwitchToState:178(854ns)}}}}}
2025-11-11T17:18:54.668377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518536904683651:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4118
2025-11-11T17:18:54.541723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518538916713229:2061] [node 4] ICP01 ready to work
2025-11-11T17:18:54.671155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518538916713245:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:54.697764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518539462928187:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.697897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518539462928125:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.723298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518538916713245:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25296003ns) {Configure:104(25290448ns) {ProcessConfigured:135(68561ns) {StartInitialHandshake:157(63023ns) {DropHandshakes:469(3107ns) {DropIncomingHandshake:441(651ns) DropOutgoingHandshake:456(677ns)} GenerateSessionVirtualId:988(1739ns) PrepareNewSessionHandshake:476(1386ns) {SwitchToState:178(306ns)}}}}}
2025-11-11T17:18:54.723685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518538916713255:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25155
2025-11-11T17:18:54.448501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518537143921138:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7786
2025-11-11T17:18:54.703979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518535907591593:2081] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.737695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518539462928199:2072] [node 5] ICP01 ready to work
2025-11-11T17:18:54.448551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518537143921139:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7786
2025-11-11T17:18:54.730214Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518538916713199:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(308859924ns) {Configure:104(308854633ns) {ProcessConfigured:135(99804168ns) {StartInitialHandshake:157(99799749ns) {DropHandshakes:469(4203ns) {DropIncomingHandshake:441(917ns) DropOutgoingHandshake:456(675ns)} GenerateSessionVirtualId:988(1140ns) PrepareNewSessionHandshake:476(5078ns) {SwitchToState:178(2988ns)}}}}}
2025-11-11T17:18:54.703656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518536996007078:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.737967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518535907591595:2083] [node 4] ICP01 ready to work
2025-11-11T17:18:54.754252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518536904683653:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4118
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.759103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518538916713229:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(123729171ns) {Configure:104(123719615ns) {ProcessConfigured:135(65846ns) {StartInitialHandshake:157(62568ns) {DropHandshakes:469(3388ns) {DropIncomingHandshake:441(701ns) DropOutgoingHandshake:456(604ns)} GenerateSessionVirtualId:988(908ns) PrepareNewSessionHandshake:476(2095ns) {SwitchToState:178(645ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.826345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518536996007062:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(400292977ns) {Configure:104(400283455ns) {ProcessConfigured:135(64699ns) {StartInitialHandshake:157(62210ns) {DropHandshakes:469(2958ns) {DropIncomingHandshake:441(753ns) DropOutgoingHandshake:456(476ns)} GenerateSessionVirtualId:988(861ns) PrepareNewSessionHandshake:476(2397ns) {SwitchToState:178(708ns)}}}}}
2025-11-11T17:18:54.827057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518536996007129:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
2025-11-11T17:18:54.827246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518536996007130:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
2025-11-11T17:18:54.829915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518536996007129:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.830097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518536996007062:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:54.477798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518537143921136:2068] [node 4] ICP01 ready to work
2025-11-11T17:18:54.841828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518536996007130:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.798059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518539462928202:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:54.766074Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683651:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.862133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518536996007133:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
2025-11-11T17:18:54.862700Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518536996007129:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:54.862878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518536996007062:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:54.862932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:54.862973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518536996007062:2049] [node 2] ICP22 created new session: [4:7571518536996007134:2049]
2025-11-11T17:18:54.863011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518536996007134:2049] [node 2] ICS09 handshake done sender: [4:7571518536996007129:2112] self: [4:7571518536996007132:0] peer: [2:7571518538916713259:0] socket: 189 qp: -1
2025-11-11T17:18:54.863043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518536996007134:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:54.863073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:54.673643Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921138:2078] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.773804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713226:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.882994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518535907591598:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.891639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518535907591599:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:54.893196Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518535907591599:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.893303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518535907591527:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.894346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518535907591601:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:54.694394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518532848953808:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(348435356ns) {Configure:104(336530821ns) {ProcessConfigured:135(64644ns) {StartInitialHandshake:157(59736ns) {DropHandshakes:469(3292ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(614ns)} GenerateSessionVirtualId:988(781ns) PrepareNewSessionHandshake:476(1830ns) {SwitchToState:178(540ns)}}}}}
2025-11-11T17:18:54.773883Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713255:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.798099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518539462928203:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:54.885692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518536996007061:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(487695930ns) {Configure:104(487692126ns) {ProcessConfigured:135(15595395ns) {StartInitialHandshake:157(15593574ns) {DropHandshakes:469(1354ns) {DropIncomingHandshake:441(354ns) DropOutgoingHandshake:456(236ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:476(3995ns) {SwitchToState:178(1530ns)}}}}}
2025-11-11T17:18:54.785712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683653:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.902294Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518535907591599:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:54.701799Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921141:2080] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.801573Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713257:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.798422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518539462928201:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:54.849794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518536904683629:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(301969275ns) {Configure:104(282056995ns) {ProcessConfigured:135(25012998ns) {StartInitialHandshake:157(25010453ns) {DropHandshakes:469(3151ns) {DropIncomingHandshake:441(781ns) DropOutgoingHandshake:456(510ns)} GenerateSessionVirtualId:988(653ns) PrepareNewSessionHandshake:476(4462ns) {SwitchToState:178(2839ns)}}}}}
2025-11-11T17:18:54.914090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518535907591526:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(499983621ns) {Configure:104(499974973ns) {ProcessConfigured:135(27936964ns) {StartInitialHandshake:157(27933515ns) {DropHandshakes:469(2930ns) {DropIncomingHandshake:441(760ns) DropOutgoingHandshake:456(604ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:476(2170ns) {SwitchToState:178(979ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.909634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518536996007063:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(280396651ns) {Configure:104(280390548ns) {ProcessConfigured:135(55745ns) {StartInitialHandshake:157(53759ns) {DropHandshakes:469(1624ns) {DropIncomingHandshake:441(405ns) DropOutgoingHandshake:456(255ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:476(1132ns) {SwitchToState:178(465ns)}}}}}
2025-11-11T17:18:54.801607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518538916713257:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:54.799336Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27166) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:54.701825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518537143921141:2080] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:54.922642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518535907591527:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:54.922698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:54.922742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518535907591527:2049] [node 2] ICP22 created new session: [5:7571518535907591604:2049]
2025-11-11T17:18:54.922781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518535907591604:2049] [node 2] ICS09 handshake done sender: [5:7571518535907591599:2113] self: [5:7571518535907591600:0] peer: [2:7571518538916713234:0] socket: 217 qp: -1
2025-11-11T17:18:54.922819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518535907591604:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:54.922850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:54.953656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518535907591598:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.964541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518535907591593:2081] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(209630613ns) {Configure:104(204386830ns) {ProcessConfigured:135(46539ns) {StartInitialHandshake:157(43784ns) {DropHandshakes:469(4576ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(242ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:476(1748ns) {SwitchToState:178(635ns)}}}}}
2025-11-11T17:18:54.886475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518532609716288:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(500558867ns) {Configure:104(500538550ns) {ProcessConfigured:135(36638489ns) {StartInitialHandshake:157(36635249ns) {DropHandshakes:469(3560ns) {DropIncomingHandshake:441(828ns) DropOutgoingHandshake:456(586ns)} GenerateSessionVirtualId:988(1253ns) PrepareNewSessionHandshake:476(4610ns) {SwitchToState:178(1438ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.921618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518536996007063:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:54.800105Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518539462928202:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.800188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518539462928125:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27166) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518539462928187:2103] held: no
2025-11-11T17:18:54.800310Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518539462928187:2103] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:54.800527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518539462928125:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:54.800563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:54.800602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518539462928125:2050] [node 3] ICP22 created new session: [6:7571518539462928204:2050]
2025-11-11T17:18:54.800638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518539462928204:2050] [node 3] ICS09 handshake done sender: [6:7571518539462928187:2103] self: [6:7571518539462928200:0] peer: [3:7571518537143921140:0] socket: 192 qp: -1
2025-11-11T17:18:54.800671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518539462928204:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:54.800700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:54.801056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518539462928124:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:54.897908Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518539462928203:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.898111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518539462928123:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:54.962297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518539462928199:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(190631089ns) {Configure:104(190622953ns) {ProcessConfigured:135(66176ns) {StartInitialHandshake:157(63504ns) {DropHandshakes:469(3208ns) {DropIncomingHandshake:441(1082ns) DropOutgoingHandshake:456(670ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:476(1839ns) {SwitchToState:178(668ns)}}}}}
2025-11-11T17:18:54.978348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518539462928212:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:54.978648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518539462928213:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:54.978725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518539462928209:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:54.980488Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518539462928202:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:54.980896Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518539462928203:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:54.981145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518539462928124:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:54.981191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:54.981236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518539462928124:2049] [node 2] ICP22 created new session: [6:7571518539462928214:2049]
2025-11-11T17:18:54.981277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518539462928214:2049] [node 2] ICS09 handshake done sender: [6:7571518539462928202:2116] self: [6:7571518539462928206:0] peer: [2:7571518538916713256:0] socket: 203 qp: -1
2025-11-11T17:18:54.981307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518539462928214:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:54.981334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:54.981859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518539462928123:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.995238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518536996007146:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
2025-11-11T17:18:54.701863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518537143921171:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7786
2025-11-11T17:18:54.828177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518538916713263:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25155
2025-11-11T17:18:55.013693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518535907591528:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:54.906207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518536904683662:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4118
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.841988Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713260:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.015281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:54.702772Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518537143921141:2080] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:54.921632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683655:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.007182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518536996007124:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(303250541ns) {Configure:104(303241954ns) {ProcessConfigured:135(13406040ns) {StartInitialHandshake:157(13402690ns) {DropHandshakes:469(2460ns) {DropIncomingHandshake:441(495ns) DropOutgoingHandshake:456(474ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:476(7752ns) {SwitchToState:178(1039ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.921677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518536904683655:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.842017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518538916713260:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:54.702990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518537143921137:2076] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:55.029107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518539462928123:2048] [node 1] ICP22 created new session: [6:7571518539462928216:2048]
2025-11-11T17:18:54.933918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518536904683622:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(455861184ns) {Configure:104(455852950ns) {ProcessConfigured:135(23427249ns) {StartInitialHandshake:157(15833539ns) {DropHandshakes:469(4038ns) {DropIncomingHandshake:441(922ns) DropOutgoingHandshake:456(554ns)} GenerateSessionVirtualId:988(1955ns) PrepareNewSessionHandshake:476(3424ns) {SwitchToState:178(1585ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:55.047553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518535907591528:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(56287310ns) {IncomingHandshake:261(56285589ns) {DropIncomingHandshake:441(517ns) PrepareNewSessionHandshake:476(3772ns) {SwitchToState:178(782ns)} GenerateSessionVirtualId:988(567ns) IssueIncomingHandshakeReply:193(34099783ns) {SwitchToState:178(3905ns)}}}
2025-11-11T17:18:55.029058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518536996007079:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(405620292ns) {Configure:104(405613778ns) {ProcessConfigured:135(21571115ns) {StartInitialHandshake:157(21569132ns) {DropHandshakes:469(2920ns) {DropIncomingHandshake:441(387ns) DropOutgoingHandshake:456(555ns)} GenerateSessionVirtualId:988(1390ns) PrepareNewSessionHandshake:476(2756ns) {SwitchToState:178(1444ns)}}}}}
2025-11-11T17:18:54.850452Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518538916713260:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.703036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:55.034184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518539462928216:2048] [node 1] ICS09 handshake done sender: [6:7571518539462928203:2117] self: [6:7571518539462928208:0] peer: [1:7571518536904683654:0] socket: 205 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.945970Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518536904683655:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:55.070181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518540202558906:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:55.053266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518539462928216:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:55.046029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518536996007130:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:54.703077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518537143921137:2076] [node 6] ICP22 created new session: [3:7571518537143921180:2076]
2025-11-11T17:18:54.890483Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713263:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:54.703114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518537143921180:2076] [node 6] ICS09 handshake done sender: [3:7571518537143921141:2080] self: [3:7571518537143921140:0] peer: [6:7571518539462928200:0] socket: 187 qp: -1
2025-11-11T17:18:54.703146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518537143921180:2076] [node 6] ICS10 traffic start
2025-11-11T17:18:54.703178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:54.761646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921171:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.780940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518537143921136:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(239003847ns) {Configure:104(238993291ns) {ProcessConfigured:135(55264ns) {StartInitialHandshake:157(52585ns) {DropHandshakes:469(3086ns) {DropIncomingHandshake:441(727ns) DropOutgoingHandshake:456(646ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:476(2502ns) {SwitchToState:178(737ns)}}}}}
2025-11-11T17:18:54.793799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518537143921137:2076] [node 6] ICP18 handshake (actor [3:7571518537143921171:2109]) is too old
2025-11-11T17:18:54.793903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [3:7571518537143921137:2076] [node 6] ICP35 event processing took too much time StateWork:293(23968958ns) {IncomingHandshake:261(16194142ns)}
2025-11-11T17:18:54.813677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921139:2079] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.881986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518537143921129:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(377850429ns) {Configure:104(377843585ns) {ProcessConfigured:135(78808ns) {StartInitialHandshake:157(75514ns) {DropHandshakes:469(3762ns) {DropIncomingHandshake:441(1177ns) DropOutgoingHandshake:456(695ns)} GenerateSessionVirtualId:988(647ns) PrepareNewSessionHandshake:476(2034ns) {SwitchToState:178(604ns)}}}}}
2025-11-11T17:18:54.965927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683650:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.965961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518536904683650:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.025837Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683662:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.025924Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683659:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.025948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518536904683659:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.025993Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518536904683650:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:55.026917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518541199650966:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4118
2025-11-11T17:18:55.029781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518536904683621:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(460076273ns) {Configure:104(460069106ns) {ProcessConfigured:135(40211103ns) {StartInitialHandshake:157(40208156ns) {DropHandshakes:469(2993ns) {DropIncomingHandshake:441(698ns) DropOutgoingHandshake:456(482ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:476(2189ns) {SwitchToState:178(730ns)}}}}}
2025-11-11T17:18:55.029928Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518536904683659:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:55.046289Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518541199650966:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.046334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518536904683629:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.046385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:55.046417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518536904683629:2071] [node 6] ICP22 created new session: [1:7571518541199650969:2071]
2025-11-11T17:18:55.046462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518541199650969:2071] [node 6] ICS09 handshake done sender: [1:7571518536904683655:2113] self: [1:7571518536904683654:0] peer: [6:7571518539462928208:0] socket: 202 qp: -1
2025-11-11T17:18:55.046494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518541199650969:2071] [node 6] ICS10 traffic start
2025-11-11T17:18:55.046526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:55.082403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518532609716287:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.082439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:55.082491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518532609716287:2048] [node 2] ICP22 created new session: [1:7571518541199650971:2048]
2025-11-11T17:18:55.082528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518541199650971:2048] [node 2] ICS09 handshake done sender: [1:7571518536904683650:2110] self: [1:7571518536904683649:0] peer: [2:7571518538916713275:0] socket: 194 qp: -1
2025-11-11T17:18:55.082557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518541199650971:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:55.082586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:55.082945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518532609716288:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.082984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:55.083020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518532609716288:2049] [node 3] ICP22 created new session: [1:7571518541199650973:2049]
2025-11-11T17:18:55.083065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518541199650973:2049] [node 3] ICS09 handshake done sender: [1:7571518536904683659:2114] self: [1:7571518536904683657:0] peer: [3:7571518537143921194:0] socket: 212 qp: -1
2025-11-11T17:18:55.083090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518541199650973:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:55.083111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:55.122170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518536904683667:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4118
2025-11-11T17:18:55.142186Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 2] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:55.149738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518532609716287:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:54.890838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518538916713266:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25155
2025-11-11T17:18:54.891160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518538916713229:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:54.893566Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713253:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.893600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518538916713253:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:54.894528Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518538916713253:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:54.913680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:54.918828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518538916713229:2061] [node 4] ICP22 created new session: [2:7571518538916713271:2061]
2025-11-11T17:18:54.924387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518538916713199:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:54.924428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:54.924467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518538916713199:2063] [node 5] ICP22 created new session: [2:7571518538916713273:2063]
2025-11-11T17:18:54.924513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518538916713273:2063] [node 5] ICS09 handshake done sender: [2:7571518538916713253:2112] self: [2:7571518538916713234:0] peer: [5:7571518535907591600:0] socket: 216 qp: -1
2025-11-11T17:18:54.924538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518538916713273:2063] [node 5] ICS10 traffic start
2025-11-11T17:18:54.924569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:54.927318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518534621745886:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(583046058ns) {Configure:104(545814420ns) {ProcessConfigured:135(23908324ns) {StartInitialHandshake:157(14599638ns) {DropHandshakes:469(2172ns) {DropIncomingHandshake:441(124ns) DropOutgoingHandshake:456(418ns)} GenerateSessionVirtualId:988(1723ns) PrepareNewSessionHandshake:476(1925ns) {SwitchToState:178(978ns)}}}}}
2025-11-11T17:18:54.937988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518538916713271:2061] [node 4] ICS09 handshake done sender: [2:7571518538916713260:2116] self: [2:7571518538916713259:0] peer: [4:7571518536996007132:0] socket: 201 qp: -1
2025-11-11T17:18:54.942213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518534621745886:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:54.949706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518538916713271:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:54.950246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518538916713199:2063] [node 5] ICP35 event processing took too much time PendingConnection:278(25651555ns) {HandleHandshakeStatus:329(25648992ns) {DropIncomingHandshake:441(9285ns) DropOutgoingHandshake:456(405ns) SwitchToState:178(159ns) ProcessPendingSessionEvents:495(25381979ns) {ForwardSessionEventToSession:612(25337142ns) ForwardSessionEventToSession:612(5535ns) ForwardSessionEventToSession:612(5142ns) ForwardSessionEventToSession:612(5371ns) ForwardSessionEventToSession:612(5936ns) ForwardSessionEventToSession:612(2784ns) ForwardSessionEventToSession:612(2113ns) ForwardSessionEventToSession:612(4715ns) ForwardSessionEventToSession:612(1830ns) ForwardSessionEventToSession:612(2047ns)}}}
2025-11-11T17:18:54.953744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:54.956416Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713266:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.971497Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518538916713257:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:54.979084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518538916713245:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:54.979174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:54.979220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518538916713245:2066] [node 6] ICP22 created new session: [2:7571518538916713279:2066]
2025-11-11T17:18:54.979278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518538916713279:2066] [node 6] ICS09 handshake done sender: [2:7571518538916713257:2115] self: [2:7571518538916713256:0] peer: [6:7571518539462928206:0] socket: 199 qp: -1
2025-11-11T17:18:54.979312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518538916713279:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:54.979353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:55.006363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518538916713229:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(115002491ns) {HandleHandshakeStatus:329(115000025ns) {DropIncomingHandshake:441(12404ns) DropOutgoingHandshake:456(417ns) SwitchToState:178(305ns) ProcessPendingSessionEvents:495(24186356ns) {ForwardSessionEventToSession:612(24133510ns) ForwardSessionEventToSession:612(6710ns) ForwardSessionEventToSession:612(9217ns) ForwardSessionEventToSession:612(2863ns) ForwardSessionEventToSession:612(2851ns) ForwardSessionEventToSession:612(6876ns) ForwardSessionEventToSession:612(3273ns) ForwardSessionEventToSession:612(2200ns) ForwardSessionEventToSession:612(4738ns) ForwardSessionEventToSession:612(3209ns)}}}
2025-11-11T17:18:55.027566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518543211680577:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25155
2025-11-11T17:18:55.050086Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518538916713201:2064] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.050104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518538916713201:2064] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.051260Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518538916713255:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:55.061788Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518538916713201:2064] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:55.078728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518534621745886:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.078793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:55.089851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518534621745886:2048] [node 1] ICP22 created new session: [2:7571518543211680579:2048]
2025-11-11T17:18:55.090263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518534621745887:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.090315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:55.090356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518534621745887:2049] [node 3] ICP22 created new session: [2:7571518543211680580:2049]
2025-11-11T17:18:55.090392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518543211680580:2049] [node 3] ICS09 handshake done sender: [2:7571518538916713201:2064] self: [2:7571518538916713200:0] peer: [3:7571518541438888492:0] socket: 176 qp: -1
2025-11-11T17:18:55.090421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518543211680580:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:55.090446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:55.092147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518543211680579:2048] [node 1] ICS09 handshake done sender: [2:7571518538916713255:2114] self: [2:7571518538916713275:0] peer: [1:7571518536904683649:0] socket: 195 qp: -1
2025-11-11T17:18:55.099883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518543211680582:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25155
2025-11-11T17:18:54.890298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518537143921189:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7786
2025-11-11T17:18:55.102023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518543211680579:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:55.113749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:54.924569Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921183:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.126280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518534621745886:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(58986747ns) {HandleHandshakeStatus:329(58984149ns) {DropIncomingHandshake:441(1919ns) DropOutgoingHandshake:456(14175ns) SwitchToState:178(692ns) ProcessPendingSessionEvents:495(89181ns) {ForwardSessionEventToSession:612(45819ns) ForwardSessionEventToSession:612(2553ns) ForwardSessionEventToSession:612(5080ns) ForwardSessionEventToSession:612(1863ns) ForwardSessionEventToSession:612(5361ns) ForwardSessionEventToSession:612(2719ns) ForwardSessionEventToSession:612(2439ns) ForwardSessionEventToSession:612(8042ns) ForwardSessionEventToSession:612(4405ns) ForwardSessionEventToSession:612(1615ns)}}}
2025-11-11T17:18:54.924590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518537143921183:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:54.945856Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921189:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:54.954167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518537143921192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7786
No RDMA context found for GID: ::ffff:127.1.0.02025-11-11T17:18:54.993827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518532848953808:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:54.994736Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518537143921183:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:54.993711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518532848953809:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(499844949ns) {Configure:104(499839136ns) {ProcessConfigured:135(19939363ns) {StartInitialHandshake:157(19936786ns) {DropHandshakes:469(2662ns) {DropIncomingHandshake:441(570ns) DropOutgoingHandshake:456(588ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:476(2125ns) {SwitchToState:178(691ns)}}}}}
2025-11-11T17:18:55.082056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518540202558908:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:55.158218Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:18:55.066486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:55.066346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518536996007063:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.101956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518540202558907:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:55.097669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:55.165920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518534621745887:2049] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:55.001977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518532848953808:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(27663623ns) {IncomingHandshake:261(27661617ns) {DropIncomingHandshake:441(2947ns) PrepareNewSessionHandshake:476(5791ns) {SwitchToState:178(4566ns)} GenerateSessionVirtualId:988(1269ns) IssueIncomingHandshakeReply:193(285427ns) {SwitchToState:178(278ns)}}}
2025-11-11T17:18:55.118098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518539462928213:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.104405Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518540202558906:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.201646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683668:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.201699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518536904683668:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.138312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518543757895515:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:55.209654Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518536904683664:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.209707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518536904683664:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.105661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518536996007063:2050] [node 3] ICP22 created new session: [4:7571518541290974446:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:55.106111Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518535907591598:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:55.105916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518541290974447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:55.027782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518537143921136:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.225199Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518536904683668:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:55.154563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518539462928123:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(172428910ns) {HandleHandshakeStatus:329(172427118ns) {DropIncomingHandshake:441(762ns) DropOutgoingHandshake:456(3778ns) SwitchToState:178(317ns) ProcessPendingSessionEvents:495(20522627ns) {ForwardSessionEventToSession:612(20465815ns) ForwardSessionEventToSession:612(5918ns) ForwardSessionEventToSession:612(2816ns) ForwardSessionEventToSession:612(7004ns) ForwardSessionEventToSession:612(9448ns) ForwardSessionEventToSession:612(5509ns) ForwardSessionEventToSession:612(3644ns) ForwardSessionEventToSession:612(2639ns) ForwardSessionEventToSession:612(5419ns) ForwardSessionEventToSession:612(2816ns)}}}
2025-11-11T17:18:55.106267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518535907591528:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:55.125681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518541290974446:2050] [node 3] ICS09 handshake done sender: [4:7571518536996007130:2113] self: [4:7571518536996007141:0] peer: [3:7571518537143921182:0] socket: 191 qp: -1
2025-11-11T17:18:55.225238Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518536904683664:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:55.251575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518536904683621:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.251655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:55.251698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518536904683621:2069] [node 5] ICP22 created new session: [1:7571518541199650980:2069]
2025-11-11T17:18:55.251741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518541199650980:2069] [node 5] ICS09 handshake done sender: [1:7571518536904683668:2118] self: [1:7571518536904683666:0] peer: [5:7571518540202558916:0] socket: 214 qp: -1
2025-11-11T17:18:55.251780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518541199650980:2069] [node 5] ICS10 traffic start
2025-11-11T17:18:55.251828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:55.252634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518536904683622:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.252673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:55.252712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518536904683622:2068] [node 4] ICP22 created new session: [1:7571518541199650982:2068]
2025-11-11T17:18:55.252745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518541199650982:2068] [node 4] ICS09 handshake done sender: [1:7571518536904683664:2116] self: [1:7571518536904683663:0] peer: [4:7571518541290974451:0] socket: 243 qp: -1
2025-11-11T17:18:55.252773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518541199650982:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:55.252797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:55.033990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518532848953809:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:55.158062Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518543757895515:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.131208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518541290974446:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:55.145795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:55.177663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518541290974450:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
2025-11-11T17:18:55.186175Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518541290974447:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.190020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518536996007061:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:55.194058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518536996007144:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.194085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518536996007144:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.194492Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518541290974450:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.198035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518536996007063:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(131441838ns) {HandleHandshakeStatus:329(131437476ns) {DropIncomingHandshake:441(797ns) DropOutgoingHandshake:456(6672ns) SwitchToState:178(527ns) ProcessPendingSessionEvents:495(31553548ns) {ForwardSessionEventToSession:612(23507352ns) ForwardSessionEventToSession:612(27455ns) ForwardSessionEventToSession:612(6707ns) ForwardSessionEventToSession:612(2794ns) ForwardSessionEventToSession:612(9517ns) ForwardSessionEventToSession:612(2705ns) ForwardSessionEventToSession:612(2247ns) ForwardSessionEventToSession:612(4488ns) ForwardSessionEventToSession:612(2536ns) ForwardSessionEventToSession:612(2356ns)}}}
2025-11-11T17:18:55.201714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518541290974443:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.201738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518541290974443:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.226221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518541290974454:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61692
2025-11-11T17:18:55.226681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518541290974447:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:55.226888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518536996007061:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.226959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:55.227022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518536996007061:2048] [node 1] ICP22 created new session: [4:7571518541290974455:2048]
2025-11-11T17:18:55.227061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518541290974455:2048] [node 1] ICS09 handshake done sender: [4:7571518541290974447:2120] self: [4:7571518541290974451:0] peer: [1:7571518536904683663:0] socket: 244 qp: -1
2025-11-11T17:18:55.227098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518541290974455:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:55.227128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:55.233851Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518536996007144:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:55.234141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518536996007079:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.234220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:55.234269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518536996007079:2063] [node 6] ICP22 created new session: [4:7571518541290974457:2063]
2025-11-11T17:18:55.234316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518541290974457:2063] [node 6] ICS09 handshake done sender: [4:7571518536996007144:2117] self: [4:7571518536996007143:0] peer: [6:7571518543757895517:0] socket: 182 qp: -1
2025-11-11T17:18:55.234347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518541290974457:2063] [node 6] ICS10 traffic start
2025-11-11T17:18:55.234380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:55.262388Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518541290974443:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:55.262579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518536996007124:2071] [node 5] ICP35 event processing took too much time PendingConnection:278(26369248ns) {IncomingHandshake:261(14209625ns) {DropIncomingHandshake:441(498ns) PrepareNewSessionHandshake:476(2237ns) {SwitchToState:178(927ns)} GenerateSessionVirtualId:988(511ns) IssueIncomingHandshakeReply:193(7254ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:55.169737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518539462928199:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:55.190533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518543757895518:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:55.193888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518539462928199:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(36059464ns) {IncomingHandshake:261(16104810ns) {DropIncomingHandshake:441(651ns) PrepareNewSessionHandshake:476(2178ns) {SwitchToState:178(1078ns)} GenerateSessionVirtualId:988(1007ns) IssueIncomingHandshakeReply:193(11459237ns) {SwitchToState:178(1587ns)}}}
2025-11-11T17:18:55.194422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518539462928139:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:55.213666Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518539462928213:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:55.221828Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:55.224776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518539462928199:2072] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518539462928213:2121] held: no
2025-11-11T17:18:55.250588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518539462928199:2072] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.250636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:55.250691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518539462928199:2072] [node 5] ICP22 created new session: [6:7571518543757895520:2072]
2025-11-11T17:18:55.250740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518543757895520:2072] [node 5] ICS09 handshake done sender: [6:7571518539462928213:2121] self: [6:7571518543757895516:0] peer: [5:7571518535907591606:0] socket: 229 qp: -1
2025-11-11T17:18:55.250777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518543757895520:2072] [node 5] ICS10 traffic start
2025-11-11T17:18:55.250815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:55.261688Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:55.265842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518543757895519:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9023
2025-11-11T17:18:55.274338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518539462928139:2060] [node 4] ICP35 event processing took too much time PendingConnection:278(32496787ns) {IncomingHandshake:261(19679723ns) {DropIncomingHandshake:441(518ns) PrepareNewSessionHandshake:476(2097ns) {SwitchToState:178(1234ns)} GenerateSessionVirtualId:988(831ns) IssueIncomingHandshakeReply:193(19652187ns) {SwitchToState:178(1320ns)}}}
2025-11-11T17:18:55.285694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518536996007124:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.285802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:55.285846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518536996007124:2071] [node 5] ICP22 created new session: [4:7571518541290974460:2071]
2025-11-11T17:18:55.285901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518541290974460:2071] [node 5] ICS09 handshake done sender: [4:7571518541290974443:2119] self: [4:7571518536996007145:0] peer: [5:7571518540202558917:0] socket: 232 qp: -1
2025-11-11T17:18:55.285941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518541290974460:2071] [node 5] ICS10 traffic start
2025-11-11T17:18:55.106343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:55.044209Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921186:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.044250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518537143921186:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.045644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:55.045716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518537143921136:2068] [node 4] ICP22 created new session: [3:7571518541438888491:2068]
2025-11-11T17:18:55.052361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518541438888494:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7786
2025-11-11T17:18:55.057833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518541438888493:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7786
2025-11-11T17:18:55.078058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518537143921192:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.078113Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518537143921189:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:55.078416Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518537143921186:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:55.081666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518541438888491:2068] [node 4] ICS09 handshake done sender: [3:7571518537143921183:2115] self: [3:7571518537143921182:0] peer: [4:7571518536996007141:0] socket: 204 qp: -1
2025-11-11T17:18:55.086087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518541438888491:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:55.086129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:55.086181Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518537143921138:2078] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:55.091256Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:55.114116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518532848953808:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.114179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:55.114220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518532848953808:2048] [node 1] ICP22 created new session: [3:7571518541438888498:2048]
2025-11-11T17:18:55.114257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518541438888498:2048] [node 1] ICS09 handshake done sender: [3:7571518537143921189:2117] self: [3:7571518537143921194:0] peer: [1:7571518536904683657:0] socket: 213 qp: -1
2025-11-11T17:18:55.114289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518541438888498:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:55.114321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:55.114562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518532848953808:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:55.114937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518537143921129:2059] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.114979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:55.115013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518537143921129:2059] [node 5] ICP22 created new session: [3:7571518541438888500:2059]
2025-11-11T17:18:55.115051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518541438888500:2059] [node 5] ICS09 handshake done sender: [3:7571518537143921186:2116] self: [3:7571518537143921185:0] peer: [5:7571518535907591609:0] socket: 206 qp: -1
2025-11-11T17:18:55.115078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518541438888500:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:55.115105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:55.117268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518532848953809:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.117305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:55.117339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518532848953809:2049] [node 2] ICP22 created new session: [3:7571518541438888502:2049]
2025-11-11T17:18:55.117371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518541438888502:2049] [node 2] ICS09 handshake done sender: [3:7571518537143921138:2078] self: [3:7571518541438888492:0] peer: [2:7571518538916713200:0] socket: 185 qp: -1
2025-11-11T17:18:55.117395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518541438888502:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:55.117420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:55.138804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518537143921136:2068] [node 4] ICP35 event processing took too much time PendingConnection:278(120877942ns) {HandleHandshakeStatus:329(110812773ns) {DropIncomingHandshake:441(191ns) DropOutgoingHandshake:456(742ns) SwitchToState:178(357ns) ProcessPendingSessionEvents:495(12742812ns) {ForwardSessionEventToSession:612(12686960ns) ForwardSessionEventToSession:612(14217ns) ForwardSessionEventToSession:612(4609ns) ForwardSessionEventToSession:612(1963ns) ForwardSessionEventToSession:612(4850ns) ForwardSessionEventToSession:612(1796ns) ForwardSessionEventToSession:612(4660ns) ForwardSessionEventToSession:612(2101ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(4649ns)}}}
2025-11-11T17:18:55.138905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518537143921136:2068] [node 4] ICP18 handshake (actor [3:7571518537143921192:2119]) is too old
2025-11-11T17:18:55.293702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:55.106380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518535907591528:2050] [node 3] ICP22 created new session: [5:7571518540202558910:2050]
2025-11-11T17:18:55.306453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518539462928139:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518543757895515:2122] held: no
2025-11-11T17:18:55.106416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518540202558910:2050] [node 3] ICS09 handshake done sender: [5:7571518535907591598:2112] self: [5:7571518535907591609:0] peer: [3:7571518537143921185:0] socket: 210 qp: -1
2025-11-11T17:18:55.106444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518540202558910:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:55.106475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:55.137916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518540202558913:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:55.142439Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518540202558907:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.158459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518535907591595:2083] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(316094226ns) {Configure:104(296183612ns) {ProcessConfigured:135(24240807ns) {StartInitialHandshake:157(24237800ns) {DropHandshakes:469(2714ns) {DropIncomingHandshake:441(368ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(825ns) PrepareNewSessionHandshake:476(2030ns) {SwitchToState:178(651ns)}}}}}
2025-11-11T17:18:55.161810Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518540202558913:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.169752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518535907591526:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:55.173849Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518535907591607:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:55.173873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518535907591607:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:55.177747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518535907591595:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:55.188966Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518535907591607:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:55.213956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518535907591593:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:55.214041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:55.214095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518535907591593:2081] [node 6] ICP22 created new session: [5:7571518540202558919:2081]
2025-11-11T17:18:55.214145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518540202558919:2081] [node 6] ICS09 handshake done sender: [5:7571518535907591607:2116] self: [5:7571518535907591606:0] peer: [6:7571518543757895516:0] socket: 228 qp: -1
2025-11-11T17:18:55.214177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518540202558919:2081] [node 6] ICS10 traffic start
2025-11-11T17:18:55.214208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:55.269756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518540202558922:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:55.274190Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:55.286437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518535907591526:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518540202558913:2121] held: no
2025-11-11T17:18:55.289742Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518540202558906:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:55.293625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518540202558921:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57864
2025-11-11T17:18:55.294168Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:55.308585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518535907591595:2083] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.308645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:55.313731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518535907591595:2083] [node 4] ICP22 created new session: [5:7571518540202558923:2083]
2025-11-11T17:18:55.321706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518540202558923:2083] [node 4] ICS09 handshake done sender: [5:7571518540202558906:2117] self: [5:7571518540202558917:0] peer: [4:7571518536996007145:0] socket: 181 qp: -1
2025-11-11T17:18:55.333717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518540202558923:2083] [node 4] ICS10 traffic start
2025-11-11T17:18:55.333799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:55.334302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518535907591595:2083] [node 4] ICP35 event processing took too much time PendingConnection:278(25513890ns) {HandleHandshakeStatus:329(25510271ns) {DropIncomingHandshake:441(1159ns) DropOutgoingHandshake:456(10339ns) SwitchToState:178(589ns) ProcessPendingSessionEvents:495(118425ns) {ForwardSessionEventToSession:612(65926ns) ForwardSessionEventToSession:612(6700ns) ForwardSessionEventToSession:612(2700ns) ForwardSessionEventToSession:612(5683ns) ForwardSessionEventToSession:612(7495ns) ForwardSessionEventToSession:612(2289ns) ForwardSessionEventToSession:612(2204ns) ForwardSessionEventToSession:612(5641ns) ForwardSessionEventToSession:612(2515ns) ForwardSessionEventToSession:612(8894ns)}}}
2025-11-11T17:18:55.334422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518535907591595:2083] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:55.357745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518535907591526:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20690502ns) {HandleHandshakeStatus:395(16279818ns) {LogHandshakeFail:472(14198ns) {UpdateErrorStateLog:544(5164ns)} DropOutgoingHandshake:456(1170ns) ProcessPendingSessionEvents:495(8529419ns) {DropSessionEvent:512(8737ns) DropSessionEvent:512(3388834ns) DropSessionEvent:512(7628ns) DropSessionEvent:512(4494ns) DropSessionEvent:512(3190ns) DropSessionEvent:512(3269ns) DropSessionEvent:512(4077ns) DropSessionEvent:512(12104ns) DropSessionEvent:512(3919ns) DropSessionEvent:512(3478ns)}}}
2025-11-11T17:18:55.357877Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518540202558913:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:55.349793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518539462928139:2060] [node 4] ICP35 event processing took too much time PendingConnection:278(23443970ns) {HandleHandshakeStatus:395(23441639ns) {LogHandshakeFail:472(3371592ns) {UpdateErrorStateLog:544(5166ns)} DropOutgoingHandshake:456(2504ns) ProcessPendingSessionEvents:495(62338ns) {DropSessionEvent:512(18144ns) DropSessionEvent:512(9748ns) DropSessionEvent:512(8667ns) DropSessionEvent:512(5571ns) DropSessionEvent:512(4901ns) DropSessionEvent:512(4336ns) DropSessionEvent:512(6400ns)}}}
2025-11-11T17:18:55.349937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518543757895515:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:55.362171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518536996007124:2071] [node 5] ICP35 event processing took too much time PendingConnection:278(64462746ns) {HandleHandshakeStatus:329(64456439ns) {DropIncomingHandshake:441(24373ns) DropOutgoingHandshake:456(567ns) SwitchToState:178(617ns) ProcessPendingSessionEvents:495(56261862ns) {ForwardSessionEventToSession:612(16917986ns) ForwardSessionEventToSession:612(3114148ns) ForwardSessionEventToSession:612(5735ns) ForwardSessionEventToSession:612(2351ns) ForwardSessionEventToSession:612(11743383ns) ForwardSessionEventToSession:612(4136473ns) ForwardSessionEventToSession:612(11404ns) ForwardSessionEventToSession:612(7984513ns) ForwardSessionEventToSession:612(2557ns) ForwardSessionEventToSession:612(12318380ns)}}}
2025-11-11T17:18:55.363853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518539462928139:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.363928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:55.363973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518539462928139:2060] [node 4] ICP22 created new session: [6:7571518543757895522:2060]
2025-11-11T17:18:55.364011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518543757895522:2060] [node 4] ICS09 handshake done sender: [6:7571518543757895515:2122] self: [6:7571518543757895517:0] peer: [4:7571518536996007143:0] socket: 196 qp: -1
2025-11-11T17:18:55.364040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518543757895522:2060] [node 4] ICS10 traffic start
2025-11-11T17:18:55.364083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:55.362965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518535907591526:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:55.363047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:55.363097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518535907591526:2048] [node 1] ICP22 created new session: [5:7571518540202558925:2048]
2025-11-11T17:18:55.363136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518540202558925:2048] [node 1] ICS09 handshake done sender: [5:7571518540202558913:2121] self: [5:7571518540202558916:0] peer: [1:7571518536904683666:0] socket: 242 qp: -1
2025-11-11T17:18:55.363171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518540202558925:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:55.363232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:55.458520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:55.458732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518541438888498:2048] [node 1] ICS07 socket disconnect 213 reason# EndOfStream
2025-11-11T17:18:55.458757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518541438888498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.458820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:55.458853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518541438888498:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:55.457711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:55.457967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518543211680579:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-11T17:18:55.458002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518534621745886:2048] [node 1] ICP30 unregister session Session# [2:7571518543211680579:2048] VirtualId# [2:7571518538916713275:0]
2025-11-11T17:18:55.458031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518543211680579:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.458122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:55.458172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:55.461736Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:18:55.461856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518532848953808:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:18:55.461884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518541438888498:2048] [node 1] ICS15 start handshake
2025-11-11T17:18:55.463037Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Couldn't connect to any resolved address
2025-11-11T17:18:55.463278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518532848953808:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:55.463324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518541438888498:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:55.463355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518532848953808:2048] [node 1] ICP30 unregister session Session# [3:7571518541438888498:2048] VirtualId# [3:7571518537143921194:0]
2025-11-11T17:18:55.463397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:55.463426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518532848953808:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:55.463438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:55.475529Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:55.475788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518540202558925:2048] [node 1] ICS01 socket: 242 reason# EndOfStream
2025-11-11T17:18:55.475822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518535907591526:2048] [node 1] ICP30 unregister session Session# [5:7571518540202558925:2048] VirtualId# [5:7571518540202558916:0]
2025-11-11T17:18:55.475853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518540202558925:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.475927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:55.475962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:55.476173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:55.476373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518541290974455:2048] [node 1] ICS01 socket: 244 reason# EndOfStream
2025-11-11T17:18:55.476404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518536996007061:2048] [node 1] ICP30 unregister session Session# [4:7571518541290974455:2048] VirtualId# [4:7571518541290974451:0]
2025-11-11T17:18:55.476426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518541290974455:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.476485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:55.476520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:55.481723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518532848953808:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:18:55.481882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:55.482104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518539462928216:2048] [node 1] ICS01 socket: 205 reason# EndOfStream
2025-11-11T17:18:55.482165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518539462928123:2048] [node 1] ICP30 unregister session Session# [6:7571518539462928216:2048] VirtualId# [6:7571518539462928208:0]
2025-11-11T17:18:55.482196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518539462928216:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.482285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:55.482327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:55.599283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:55.599516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518541438888502:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-11T17:18:55.599550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518532848953809:2049] [node 2] ICP30 unregister session Session# [3:7571518541438888502:2049] VirtualId# [3:7571518541438888492:0]
2025-11-11T17:18:55.599580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518541438888502:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.599644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:55.599679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:55.612802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:55.613022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518536996007134:2049] [node 2] ICS01 socket: 189 reason# EndOfStream
2025-11-11T17:18:55.613059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518536996007062:2049] [node 2] ICP30 unregister session Session# [4:7571518536996007134:2049] VirtualId# [4:7571518536996007132:0]
2025-11-11T17:18:55.613092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518536996007134:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.613179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:55.613220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:55.613728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:55.613702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:55.614144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518539462928214:2049] [node 2] ICS07 socket disconnect 203 reason# EndOfStream
2025-11-11T17:18:55.614174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518539462928214:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.614244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:55.614288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518539462928214:2049] [node 2] ICS15 start handshake
2025-11-11T17:18:55.620565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518535907591604:2049] [node 2] ICS01 socket: 217 reason# EndOfStream
2025-11-11T17:18:55.620633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518535907591527:2049] [node 2] ICP30 unregister session Session# [5:7571518535907591604:2049] VirtualId# [5:7571518535907591600:0]
2025-11-11T17:18:55.620665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518535907591604:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.620782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:55.620826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:55.621999Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17250) Couldn't connect to any resolved address
2025-11-11T17:18:55.622375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518539462928124:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17250) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:55.622445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518539462928214:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:55.622474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518539462928124:2049] [node 2] ICP30 unregister session Session# [6:7571518539462928214:2049] VirtualId# [6:7571518539462928206:0]
2025-11-11T17:18:55.622520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:55.622555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518539462928124:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:55.622592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:55.640729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518539462928124:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:18:55.701733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:55.701975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518541290974446:2050] [node 3] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-11T17:18:55.702002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518541290974446:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.702073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:55.702109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518541290974446:2050] [node 3] ICS15 start handshake
2025-11-11T17:18:55.702841Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27166) Couldn't connect to any resolved address
2025-11-11T17:18:55.703072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518536996007063:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27166) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:18:55.703108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518541290974446:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:18:55.703138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518536996007063:2050] [node 3] ICP30 unregister session Session# [4:7571518541290974446:2050] VirtualId# [4:7571518536996007141:0]
2025-11-11T17:18:55.703172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:55.703196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518536996007063:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:55.703222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:18:55.701736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:55.702294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518539462928204:2050] [node 3] ICS01 socket: 192 reason# EndOfStream
2025-11-11T17:18:55.702326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518539462928125:2050] [node 3] ICP30 unregister session Session# [6:7571518539462928204:2050] VirtualId# [6:7571518539462928200:0]
2025-11-11T17:18:55.702354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518539462928204:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.702429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:55.702470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:55.705736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:55.706014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518540202558910:2050] [node 3] ICS01 socket: 210 reason# EndOfStream
2025-11-11T17:18:55.706078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518535907591528:2050] [node 3] ICP30 unregister session Session# [5:7571518540202558910:2050] VirtualId# [5:7571518535907591609:0]
2025-11-11T17:18:55.706118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518540202558910:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.706245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:55.706314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:55.717680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518536996007063:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:18:55.793789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:55.794038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518543757895522:2060] [node 4] ICS01 socket: 196 reason# EndOfStream
2025-11-11T17:18:55.794072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518539462928139:2060] [node 4] ICP30 unregister session Session# [6:7571518543757895522:2060] VirtualId# [6:7571518543757895517:0]
2025-11-11T17:18:55.794104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518543757895522:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.794202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:55.794249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:55.794336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:55.794543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518540202558923:2083] [node 4] ICS01 socket: 181 reason# EndOfStream
2025-11-11T17:18:55.794574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518535907591595:2083] [node 4] ICP30 unregister session Session# [5:7571518540202558923:2083] VirtualId# [5:7571518540202558917:0]
2025-11-11T17:18:55.794597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518540202558923:2083] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.794655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:55.794689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:55.881741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:55.881979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518543757895520:2072] [node 5] ICS01 socket: 229 reason# EndOfStream
2025-11-11T17:18:55.882041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518539462928199:2072] [node 5] ICP30 unregister session Session# [6:7571518543757895520:2072] VirtualId# [6:7571518543757895516:0]
2025-11-11T17:18:55.882081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518543757895520:2072] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:55.882216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:55.882265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:56.109820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518547641277554:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:56.109872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518547641277555:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:56.146007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518547734421473:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:56.146049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518547734421474:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:56.229805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518545865485592:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:56.229848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518545865485593:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:56.309902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518546386305942:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:56.309955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518546386305943:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:56.309980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518546386305944:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:56.341865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518547035659154:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:56.341911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518547035659155:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:56.341937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518547035659156:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:56.579072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518544429123006:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:56.579170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518544429123007:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:56.579202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518544429123008:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:56.579880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518544429123022:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:56.589954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518545865485618:2060] [node 5] ICP01 ready to work
2025-11-11T17:18:56.633763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518547734421533:2059] [node 5] ICP01 ready to work
2025-11-11T17:18:56.818996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518546386305942:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(249167102ns) {Configure:104(249154929ns) {ProcessConfigured:135(67890ns) {StartInitialHandshake:157(64336ns) {DropHandshakes:469(3602ns) {DropIncomingHandshake:441(911ns) DropOutgoingHandshake:456(991ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(2496ns) {SwitchToState:178(964ns)}}}}}
2025-11-11T17:18:56.749863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518547734421536:2062] [node 4] ICP01 ready to work
2025-11-11T17:18:56.805707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518547641277617:2069] [node 4] ICP01 ready to work
2025-11-11T17:18:56.817100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518545865485652:2061] [node 4] ICP01 ready to work
2025-11-11T17:18:56.819457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518546386306002:2062] [node 6] ICP01 ready to work
2025-11-11T17:18:56.817278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518544429123067:2067] [node 4] ICP01 ready to work
2025-11-11T17:18:56.819692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518546386306003:2065] [node 5] ICP01 ready to work
2025-11-11T17:18:56.838754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518546386306002:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17470242ns) {Configure:104(17467020ns) {ProcessConfigured:135(65531ns) {StartInitialHandshake:157(62484ns) {DropHandshakes:469(3027ns) {DropIncomingHandshake:441(706ns) DropOutgoingHandshake:456(613ns)} GenerateSessionVirtualId:988(1080ns) PrepareNewSessionHandshake:476(2049ns) {SwitchToState:178(669ns)}}}}}
2025-11-11T17:18:56.854052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518547734421538:2066] [node 6] ICP01 ready to work
2025-11-11T17:18:56.890321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518547641277618:2081] [node 6] ICP01 ready to work
2025-11-11T17:18:56.890672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518547641277619:2091] [node 5] ICP01 ready to work
2025-11-11T17:18:56.894603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518547035659155:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(404664810ns) {Configure:104(404660771ns) {ProcessConfigured:135(42854ns) {StartInitialHandshake:157(40834ns) {DropHandshakes:469(1092ns) {DropIncomingHandshake:441(174ns) DropOutgoingHandshake:456(232ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:476(738ns) {SwitchToState:178(292ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:56.895085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518547035659219:2068] [node 4] ICP01 ready to work
2025-11-11T17:18:56.895361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518547035659220:2075] [node 6] ICP01 ready to work
2025-11-11T17:18:56.905040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518547035659221:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:56.909960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518547641277554:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16328195ns) {Configure:104(16325896ns) {ProcessConfigured:135(42579ns) {StartInitialHandshake:157(40792ns) {DropHandshakes:469(1110ns) {DropIncomingHandshake:441(305ns) DropOutgoingHandshake:456(134ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:476(1207ns) {SwitchToState:178(514ns)}}}}}
2025-11-11T17:18:56.913084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518547641277620:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25673
2025-11-11T17:18:56.913149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518547641277621:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25673
2025-11-11T17:18:56.913362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518547641277630:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25673
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:56.937935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518546386306020:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-11T17:18:56.947548Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277621:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:56.967225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518547734421473:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(72860989ns) {Configure:104(72855008ns) {ProcessConfigured:135(39886ns) {StartInitialHandshake:157(38398ns) {DropHandshakes:469(1548ns) {DropIncomingHandshake:441(519ns) DropOutgoingHandshake:456(275ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:476(1006ns) {SwitchToState:178(230ns)}}}}}
2025-11-11T17:18:56.982322Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277630:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:56.962422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518544429123007:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(28397424ns) {Configure:104(28387356ns) {ProcessConfigured:135(66723ns) {StartInitialHandshake:157(63429ns) {DropHandshakes:469(3754ns) {DropIncomingHandshake:441(959ns) DropOutgoingHandshake:456(1105ns)} GenerateSessionVirtualId:988(768ns) PrepareNewSessionHandshake:476(1970ns) {SwitchToState:178(394ns)}}}}}
2025-11-11T17:18:56.995158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518547734421536:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27658800ns) {Configure:104(27650900ns) {ProcessConfigured:135(80755ns) {StartInitialHandshake:157(78323ns) {DropHandshakes:469(3483ns) {DropIncomingHandshake:441(826ns) DropOutgoingHandshake:456(586ns)} GenerateSessionVirtualId:988(1422ns) PrepareNewSessionHandshake:476(2614ns) {SwitchToState:178(690ns)}}}}}
2025-11-11T17:18:56.945859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518545865485593:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(34403253ns) {Configure:104(34398088ns) {ProcessConfigured:135(32893917ns) {StartInitialHandshake:157(32891194ns) {DropHandshakes:469(924ns) {DropIncomingHandshake:441(185ns) DropOutgoingHandshake:456(104ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:476(4419ns) {SwitchToState:178(1869ns)}}}}}
2025-11-11T17:18:57.001800Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518546386306020:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:56.946293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518545865485660:2094] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.014658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518545865485592:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(60753487ns) {Configure:104(60748676ns) {ProcessConfigured:135(60004ns) {StartInitialHandshake:157(57464ns) {DropHandshakes:469(2050ns) {DropIncomingHandshake:441(307ns) DropOutgoingHandshake:456(682ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:476(591ns) {SwitchToState:178(368ns)}}}}}
2025-11-11T17:18:57.017364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518545865485656:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58148
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.022817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518550160452963:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58148
2025-11-11T17:18:57.022976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518550160452964:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58148
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.030967Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518550160452964:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.017831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518546386305942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.045897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518547734421533:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35717286ns) {Configure:104(35712583ns) {ProcessConfigured:135(15083149ns) {StartInitialHandshake:157(15079939ns) {DropHandshakes:469(1434ns) {DropIncomingHandshake:441(337ns) DropOutgoingHandshake:456(190ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:476(7614ns) {SwitchToState:178(1655ns)}}}}}
2025-11-11T17:18:57.041718Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277623:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.026258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518550681273318:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-11T17:18:57.031639Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518550681273318:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.031725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518546386305944:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.063537Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518545865485662:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.063581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518545865485662:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.074702Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518550160452963:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.075263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518545865485592:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.075374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518550160452971:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58148
2025-11-11T17:18:57.075969Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518545865485662:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:57.047089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518547734421547:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31279
2025-11-11T17:18:57.047267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518547734421548:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31279
2025-11-11T17:18:57.047392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518547734421549:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31279
2025-11-11T17:18:57.047583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518552029388848:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31279
2025-11-11T17:18:57.057913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518547734421548:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.069211Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518547734421547:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.069286Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518547734421549:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.069623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518547734421473:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.041780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518547641277623:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.062990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518551936244934:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25673
2025-11-11T17:18:57.063102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518551936244935:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25673
2025-11-11T17:18:57.066060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518547641277623:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:57.066365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518547641277617:2069] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.066427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:57.066471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518547641277617:2069] [node 4] ICP22 created new session: [1:7571518551936244937:2069]
2025-11-11T17:18:57.066513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518551936244937:2069] [node 4] ICS09 handshake done sender: [1:7571518547641277623:2112] self: [1:7571518547641277622:0] peer: [4:7571518550681273317:0] socket: 199 qp: -1
2025-11-11T17:18:57.066546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518551936244937:2069] [node 4] ICS10 traffic start
2025-11-11T17:18:57.066581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:57.110397Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518551936244935:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.110498Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277627:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.110515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518547641277627:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.111019Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518551936244934:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.111088Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277625:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.111100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518547641277625:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.085787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518552029388848:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.090755Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.112328Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518550160452971:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.122625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518544429123084:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.122738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090381:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.130365Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518547734421544:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.154371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518546386305942:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518546386306020:2115] held: no
2025-11-11T17:18:57.154809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518550681273320:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-11T17:18:57.112391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518545865485652:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.152125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626529:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.152500Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518547035659221:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.152567Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518551330626529:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.152726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518547035659156:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.129938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277620:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.134475Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518547641277627:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:57.155146Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518547641277625:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.112433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:57.112482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518545865485652:2061] [node 4] ICP22 created new session: [3:7571518550160452974:2061]
2025-11-11T17:18:57.112522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518550160452974:2061] [node 4] ICS09 handshake done sender: [3:7571518545865485662:2112] self: [3:7571518545865485661:0] peer: [4:7571518550681273319:0] socket: 211 qp: -1
2025-11-11T17:18:57.112567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518550160452974:2061] [node 4] ICS10 traffic start
2025-11-11T17:18:57.112635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:57.113248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518545865485593:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.138110Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518545865485656:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.139486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518550160452978:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58148
2025-11-11T17:18:57.152925Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518550160452962:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.152946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518550160452962:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.155585Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518550160452962:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.162255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626532:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.162828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518550681273321:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-11T17:18:57.166248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518545865485618:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.166317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:57.166354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518545865485618:2060] [node 5] ICP22 created new session: [3:7571518550160452982:2060]
2025-11-11T17:18:57.166397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518550160452982:2060] [node 5] ICS09 handshake done sender: [3:7571518550160452962:2114] self: [3:7571518550160452961:0] peer: [5:7571518551330626531:0] socket: 228 qp: -1
2025-11-11T17:18:57.166426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518550160452982:2060] [node 5] ICS10 traffic start
2025-11-11T17:18:57.166456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:57.167604Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518550160452963:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.167892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518550160452981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58148
2025-11-11T17:18:57.167965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518545865485592:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.167995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:57.168030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518545865485592:2048] [node 1] ICP22 created new session: [3:7571518550160452985:2048]
2025-11-11T17:18:57.168066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518550160452985:2048] [node 1] ICS09 handshake done sender: [3:7571518550160452963:2115] self: [3:7571518550160452973:0] peer: [1:7571518547641277624:0] socket: 208 qp: -1
2025-11-11T17:18:57.168091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518550160452985:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:57.168131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.165699Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518544429123084:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.181904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518544429123067:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.182469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090383:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.130394Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518547734421544:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.169865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518550681273322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-11T17:18:57.185840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518547641277554:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.185910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:57.185955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518547641277554:2048] [node 2] ICP22 created new session: [1:7571518551936244945:2048]
2025-11-11T17:18:57.185995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518551936244945:2048] [node 2] ICS09 handshake done sender: [1:7571518547641277627:2114] self: [1:7571518547641277626:0] peer: [2:7571518552029388854:0] socket: 203 qp: -1
2025-11-11T17:18:57.186028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518551936244945:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:57.186065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:57.186452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518547641277555:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.186483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:57.186518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518547641277555:2049] [node 3] ICP22 created new session: [1:7571518551936244947:2049]
2025-11-11T17:18:57.186548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518551936244947:2049] [node 3] ICS09 handshake done sender: [1:7571518547641277625:2113] self: [1:7571518547641277624:0] peer: [3:7571518550160452973:0] socket: 200 qp: -1
2025-11-11T17:18:57.186565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518551936244947:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:57.186585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:57.159435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518552029388856:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31279
2025-11-11T17:18:57.197843Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518547734421549:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:57.199970Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.201328Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518547734421544:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:57.202476Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.203549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518547035659156:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518551330626529:2116] held: no
2025-11-11T17:18:57.203672Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518551330626529:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:57.205788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518548724090381:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.206016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518544429123006:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.206425Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277629:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.206455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518547641277629:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.207507Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518547641277629:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:57.207829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518547641277618:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.207884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:57.207925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518547641277618:2081] [node 6] ICP22 created new session: [1:7571518551936244951:2081]
2025-11-11T17:18:57.207962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518551936244951:2081] [node 6] ICS09 handshake done sender: [1:7571518547641277629:2115] self: [1:7571518547641277628:0] peer: [6:7571518548724090384:0] socket: 225 qp: -1
2025-11-11T17:18:57.207993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518551936244951:2081] [node 6] ICS10 traffic start
2025-11-11T17:18:57.208052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.213955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518547734421473:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.214016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:57.214055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518547734421473:2048] [node 1] ICP22 created new session: [2:7571518552029388858:2048]
2025-11-11T17:18:57.214095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518552029388858:2048] [node 1] ICS09 handshake done sender: [2:7571518547734421549:2116] self: [2:7571518552029388854:0] peer: [1:7571518547641277626:0] socket: 207 qp: -1
2025-11-11T17:18:57.214123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518552029388858:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:57.214156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:57.214383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518547734421473:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:57.214663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518547734421474:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.214701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:57.214745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518547734421474:2049] [node 3] ICP22 created new session: [2:7571518552029388860:2049]
2025-11-11T17:18:57.214780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518552029388860:2049] [node 3] ICS09 handshake done sender: [2:7571518547734421544:2112] self: [2:7571518547734421543:0] peer: [3:7571518550160452977:0] socket: 217 qp: -1
2025-11-11T17:18:57.214804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518552029388860:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:57.214826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:57.215604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518547035659156:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.215666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:57.215700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518547035659156:2050] [node 3] ICP22 created new session: [5:7571518551330626534:2050]
2025-11-11T17:18:57.205449Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518550681273320:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.215736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518551330626534:2050] [node 3] ICS09 handshake done sender: [5:7571518551330626529:2116] self: [5:7571518551330626531:0] peer: [3:7571518550160452961:0] socket: 229 qp: -1
2025-11-11T17:18:57.222521Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.205557Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518546386306009:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.205576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518546386306009:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.229318Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518546386306020:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.229850Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.215778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518551330626534:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:57.215804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:57.216528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626533:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.234543Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518551330626533:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.235011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626537:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.235129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626538:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.235210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518547035659154:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.235625Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518550681273318:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.237079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518544429123006:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518548724090381:2116] held: no
2025-11-11T17:18:57.237307Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518548724090383:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.237627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090385:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.238116Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518550160452971:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:57.238167Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518551330626537:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.238303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518547035659155:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.242943Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518552029388847:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.242969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518552029388847:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.242698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518546386305942:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.241833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518547641277632:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.242755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626541:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.241864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518547641277632:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.242753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.280518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090386:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.280665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518544429123008:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.280871Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518548724090381:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:57.253763Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518552029388847:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:57.253985Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518547641277632:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:57.262250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518547641277619:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.262307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:57.262349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518547641277619:2091] [node 5] ICP22 created new session: [1:7571518551936244954:2091]
2025-11-11T17:18:57.262386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518551936244954:2091] [node 5] ICS09 handshake done sender: [1:7571518547641277632:2117] self: [1:7571518547641277631:0] peer: [5:7571518551330626539:0] socket: 218 qp: -1
2025-11-11T17:18:57.262422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518551936244954:2091] [node 5] ICS10 traffic start
2025-11-11T17:18:57.262453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:57.253634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626542:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.263082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518545865485593:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518550160452971:2118] held: no
2025-11-11T17:18:57.263187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518545865485593:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.263235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:57.263275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518545865485593:2049] [node 2] ICP22 created new session: [3:7571518550160452987:2049]
2025-11-11T17:18:57.263309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518550160452987:2049] [node 2] ICS09 handshake done sender: [3:7571518550160452971:2118] self: [3:7571518550160452977:0] peer: [2:7571518547734421543:0] socket: 194 qp: -1
2025-11-11T17:18:57.263335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518550160452987:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:57.263364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:57.242792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518546386305942:2048] [node 1] ICP22 created new session: [4:7571518550681273326:2048]
2025-11-11T17:18:57.242958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518550681273326:2048] [node 1] ICS09 handshake done sender: [4:7571518546386306020:2115] self: [4:7571518550681273317:0] peer: [1:7571518547641277622:0] socket: 201 qp: -1
2025-11-11T17:18:57.242996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518550681273326:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:57.243031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:57.250305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518546386305944:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.250354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:57.250407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518546386305944:2050] [node 3] ICP22 created new session: [4:7571518550681273328:2050]
2025-11-11T17:18:57.250458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518550681273328:2050] [node 3] ICS09 handshake done sender: [4:7571518550681273318:2116] self: [4:7571518550681273319:0] peer: [3:7571518545865485661:0] socket: 213 qp: -1
2025-11-11T17:18:57.250497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518550681273328:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:57.250529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:57.251139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518546386305943:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.251831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518550681273325:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-11T17:18:57.261742Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518546386306009:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:57.281710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518550681273325:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.281859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518546386306002:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.281885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:57.281925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518546386306002:2062] [node 6] ICP22 created new session: [4:7571518550681273331:2062]
2025-11-11T17:18:57.281961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518550681273331:2062] [node 6] ICS09 handshake done sender: [4:7571518546386306009:2110] self: [4:7571518546386306008:0] peer: [6:7571518548724090382:0] socket: 192 qp: -1
2025-11-11T17:18:57.282004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518550681273331:2062] [node 6] ICS10 traffic start
2025-11-11T17:18:57.282031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:57.278621Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518547734421542:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.278649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518547734421542:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.279343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518547734421533:2059] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.279396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:57.279444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518547734421533:2059] [node 5] ICP22 created new session: [2:7571518552029388865:2059]
2025-11-11T17:18:57.279492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518552029388865:2059] [node 5] ICS09 handshake done sender: [2:7571518552029388847:2117] self: [2:7571518552029388846:0] peer: [5:7571518551330626540:0] socket: 236 qp: -1
2025-11-11T17:18:57.279522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518552029388865:2059] [node 5] ICS10 traffic start
2025-11-11T17:18:57.279556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:57.284702Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.285936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518544429123006:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.285975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:57.286008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518544429123006:2048] [node 1] ICP22 created new session: [6:7571518548724090390:2048]
2025-11-11T17:18:57.286047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518548724090390:2048] [node 1] ICS09 handshake done sender: [6:7571518548724090381:2116] self: [6:7571518548724090384:0] peer: [1:7571518547641277628:0] socket: 232 qp: -1
2025-11-11T17:18:57.286087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518548724090390:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:57.286119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:57.286438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090388:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.286700Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518550160452966:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.286728Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518550160452966:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.288055Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518550160452966:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:57.288342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518545865485660:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.288397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:57.288431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518545865485660:2094] [node 6] ICP22 created new session: [3:7571518550160452990:2094]
2025-11-11T17:18:57.288462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518550160452990:2094] [node 6] ICS09 handshake done sender: [3:7571518550160452966:2117] self: [3:7571518550160452965:0] peer: [6:7571518548724090389:0] socket: 209 qp: -1
2025-11-11T17:18:57.288488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518550160452990:2094] [node 6] ICS10 traffic start
2025-11-11T17:18:57.288543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:57.289567Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518551330626538:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.289707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518547035659154:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518551330626533:2118] held: no
2025-11-11T17:18:57.289835Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518551330626537:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:57.290642Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518544429123084:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.291566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518552029388868:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.292026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518550681273334:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-11T17:18:57.292618Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518551330626533:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:57.294114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518548724090386:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.294237Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.295316Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518547734421542:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:57.295812Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518550681273320:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:57.296047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518546386305943:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.296099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:57.296139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518546386305943:2049] [node 2] ICP22 created new session: [4:7571518550681273335:2049]
2025-11-11T17:18:57.296180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518550681273335:2049] [node 2] ICS09 handshake done sender: [4:7571518550681273320:2117] self: [4:7571518550681273330:0] peer: [2:7571518547734421541:0] socket: 222 qp: -1
2025-11-11T17:18:57.296207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518550681273335:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:57.296260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:57.297688Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.298739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518547734421536:2062] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.298835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:57.298871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518547734421536:2062] [node 4] ICP22 created new session: [2:7571518552029388869:2062]
2025-11-11T17:18:57.298910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518552029388869:2062] [node 4] ICS09 handshake done sender: [2:7571518547734421542:2111] self: [2:7571518547734421541:0] peer: [4:7571518550681273330:0] socket: 216 qp: -1
2025-11-11T17:18:57.298940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518552029388869:2062] [node 4] ICS10 traffic start
2025-11-11T17:18:57.298999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:57.300501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518547035659155:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.300575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:57.300608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518547035659155:2049] [node 2] ICP22 created new session: [5:7571518551330626543:2049]
2025-11-11T17:18:57.300644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518551330626543:2049] [node 2] ICS09 handshake done sender: [5:7571518551330626537:2119] self: [5:7571518551330626540:0] peer: [2:7571518552029388846:0] socket: 238 qp: -1
2025-11-11T17:18:57.300673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518551330626543:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:57.300726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:57.300894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518547035659155:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:57.301170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518547035659154:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.301217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:57.301275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518547035659154:2048] [node 1] ICP22 created new session: [5:7571518551330626545:2048]
2025-11-11T17:18:57.301307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518551330626545:2048] [node 1] ICS09 handshake done sender: [5:7571518551330626533:2118] self: [5:7571518551330626539:0] peer: [1:7571518547641277631:0] socket: 193 qp: -1
2025-11-11T17:18:57.301330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518551330626545:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:57.301352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.305830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518544429123067:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.305883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:57.305928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518544429123067:2067] [node 4] ICP22 created new session: [6:7571518548724090393:2067]
2025-11-11T17:18:57.305962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518548724090393:2067] [node 4] ICS09 handshake done sender: [6:7571518544429123084:2115] self: [6:7571518548724090382:0] peer: [4:7571518546386306008:0] socket: 204 qp: -1
2025-11-11T17:18:57.305989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518548724090393:2067] [node 4] ICS10 traffic start
2025-11-11T17:18:57.306016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:57.306472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090392:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.307118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518547035659219:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.314092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518544429123008:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518548724090383:2117] held: no
2025-11-11T17:18:57.314733Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518548724090383:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:57.315075Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518552029388868:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.316044Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518546386306011:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.316077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518546386306011:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.317175Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518546386306011:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.318623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518551330626549:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61425
2025-11-11T17:18:57.319027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518546386306003:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.319087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:57.319131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518546386306003:2065] [node 5] ICP22 created new session: [4:7571518550681273338:2065]
2025-11-11T17:18:57.319168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518550681273338:2065] [node 5] ICS09 handshake done sender: [4:7571518546386306011:2111] self: [4:7571518546386306010:0] peer: [5:7571518551330626548:0] socket: 237 qp: -1
2025-11-11T17:18:57.319201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518550681273338:2065] [node 5] ICS10 traffic start
2025-11-11T17:18:57.319228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.320261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518544429123007:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.320371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518544429123008:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.320430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:57.320478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518544429123008:2050] [node 3] ICP22 created new session: [6:7571518548724090397:2050]
2025-11-11T17:18:57.320519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518548724090397:2050] [node 3] ICS09 handshake done sender: [6:7571518548724090383:2117] self: [6:7571518548724090389:0] peer: [3:7571518550160452965:0] socket: 212 qp: -1
2025-11-11T17:18:57.320545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518548724090397:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:57.320572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:57.321181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090396:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.321873Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518547734421546:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.321902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518547734421546:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.322824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518547734421546:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:57.323082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518547734421538:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.323173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:57.323213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518547734421538:2066] [node 6] ICP22 created new session: [2:7571518552029388873:2066]
2025-11-11T17:18:57.323251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518552029388873:2066] [node 6] ICS09 handshake done sender: [2:7571518547734421546:2113] self: [2:7571518547734421545:0] peer: [6:7571518548724090395:0] socket: 214 qp: -1
2025-11-11T17:18:57.323275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518552029388873:2066] [node 6] ICS10 traffic start
2025-11-11T17:18:57.323300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:57.325128Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.326371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518544429123007:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518548724090386:2119] held: no
2025-11-11T17:18:57.326574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090399:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.326959Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518548724090386:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:57.327165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518544429123007:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.327227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:57.327260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518544429123007:2049] [node 2] ICP22 created new session: [6:7571518548724090400:2049]
2025-11-11T17:18:57.327296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518548724090400:2049] [node 2] ICS09 handshake done sender: [6:7571518548724090386:2119] self: [6:7571518548724090395:0] peer: [2:7571518547734421545:0] socket: 186 qp: -1
2025-11-11T17:18:57.327325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518548724090400:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:57.327372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:57.329943Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:57.330985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518547035659219:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518551330626538:2120] held: no
2025-11-11T17:18:57.331115Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518551330626538:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:57.331250Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518548724090396:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.331362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518544429123022:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:57.337855Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518547035659229:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:57.337888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518547035659229:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:57.338554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518547035659219:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.338625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:57.338679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518547035659219:2068] [node 4] ICP22 created new session: [5:7571518551330626550:2068]
2025-11-11T17:18:57.338720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518551330626550:2068] [node 4] ICS09 handshake done sender: [5:7571518551330626538:2120] self: [5:7571518551330626548:0] peer: [4:7571518546386306010:0] socket: 239 qp: -1
2025-11-11T17:18:57.338756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518551330626550:2068] [node 4] ICS10 traffic start
2025-11-11T17:18:57.338796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:57.340346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518548724090403:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43542
2025-11-11T17:18:57.340648Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518547035659229:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:57.341168Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518548724090396:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:57.341393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518544429123022:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:57.341461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:57.341500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518544429123022:2062] [node 5] ICP22 created new session: [6:7571518548724090404:2062]
2025-11-11T17:18:57.341537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518548724090404:2062] [node 5] ICS09 handshake done sender: [6:7571518548724090396:2122] self: [6:7571518548724090402:0] peer: [5:7571518547035659228:0] socket: 196 qp: -1
2025-11-11T17:18:57.341565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518548724090404:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:57.341612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:57.343062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518547035659220:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:57.343140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:57.343177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518547035659220:2075] [node 6] ICP22 created new session: [5:7571518551330626553:2075]
2025-11-11T17:18:57.343216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518551330626553:2075] [node 6] ICS09 handshake done sender: [5:7571518547035659229:2115] self: [5:7571518547035659228:0] peer: [6:7571518548724090402:0] socket: 187 qp: -1
2025-11-11T17:18:57.343239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518551330626553:2075] [node 6] ICS10 traffic start
2025-11-11T17:18:57.343262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:57.370187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:57.370410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518552029388858:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-11-11T17:18:57.370446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518547734421473:2048] [node 1] ICP30 unregister session Session# [2:7571518552029388858:2048] VirtualId# [2:7571518552029388854:0]
2025-11-11T17:18:57.370479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518552029388858:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.370553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:57.370590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:57.375524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:57.375748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518550160452985:2048] [node 1] ICS01 socket: 208 reason# EndOfStream
2025-11-11T17:18:57.375783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518545865485592:2048] [node 1] ICP30 unregister session Session# [3:7571518550160452985:2048] VirtualId# [3:7571518550160452973:0]
2025-11-11T17:18:57.375813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518550160452985:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.375895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:57.375934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:57.383368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:57.383666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518548724090390:2048] [node 1] ICS01 socket: 232 reason# EndOfStream
2025-11-11T17:18:57.383698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518544429123006:2048] [node 1] ICP30 unregister session Session# [6:7571518548724090390:2048] VirtualId# [6:7571518548724090384:0]
2025-11-11T17:18:57.383721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518548724090390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.383783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:57.383825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:57.384197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:57.384378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518551330626545:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-11-11T17:18:57.384402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518547035659154:2048] [node 1] ICP30 unregister session Session# [5:7571518551330626545:2048] VirtualId# [5:7571518551330626539:0]
2025-11-11T17:18:57.384422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518551330626545:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.384474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:57.384502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:57.382721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:57.382936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518550681273326:2048] [node 1] ICS01 socket: 201 reason# EndOfStream
2025-11-11T17:18:57.382987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518546386305942:2048] [node 1] ICP30 unregister session Session# [4:7571518550681273326:2048] VirtualId# [4:7571518550681273317:0]
2025-11-11T17:18:57.383019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518550681273326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.383087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:57.383124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:57.417264Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:57.417474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518550160452987:2049] [node 2] ICS01 socket: 194 reason# EndOfStream
2025-11-11T17:18:57.417518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518545865485593:2049] [node 2] ICP30 unregister session Session# [3:7571518550160452987:2049] VirtualId# [3:7571518550160452977:0]
2025-11-11T17:18:57.417549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518550160452987:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.417884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:57.417944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:57.429770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:57.429994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518550681273335:2049] [node 2] ICS01 socket: 222 reason# EndOfStream
2025-11-11T17:18:57.430030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518546386305943:2049] [node 2] ICP30 unregister session Session# [4:7571518550681273335:2049] VirtualId# [4:7571518550681273330:0]
2025-11-11T17:18:57.430065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518550681273335:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.430155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:57.430201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:57.432096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:57.432356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518551330626543:2049] [node 2] ICS01 socket: 238 reason# EndOfStream
2025-11-11T17:18:57.432402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518547035659155:2049] [node 2] ICP30 unregister session Session# [5:7571518551330626543:2049] VirtualId# [5:7571518551330626540:0]
2025-11-11T17:18:57.432446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518551330626543:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.432555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:57.432624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:57.432738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:57.432956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518548724090400:2049] [node 2] ICS01 socket: 186 reason# EndOfStream
2025-11-11T17:18:57.432999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518544429123007:2049] [node 2] ICP30 unregister session Session# [6:7571518548724090400:2049] VirtualId# [6:7571518548724090395:0]
2025-11-11T17:18:57.433033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518548724090400:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.433093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:57.433152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:57.509754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:57.509961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518548724090397:2050] [node 3] ICS01 socket: 212 reason# EndOfStream
2025-11-11T17:18:57.509994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518544429123008:2050] [node 3] ICP30 unregister session Session# [6:7571518548724090397:2050] VirtualId# [6:7571518548724090389:0]
2025-11-11T17:18:57.510021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518548724090397:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.510110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:57.510152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:57.509764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:57.509964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518551330626534:2050] [node 3] ICS01 socket: 229 reason# EndOfStream
2025-11-11T17:18:57.509996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518547035659156:2050] [node 3] ICP30 unregister session Session# [5:7571518551330626534:2050] VirtualId# [5:7571518551330626531:0]
2025-11-11T17:18:57.510022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518551330626534:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.510106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:57.510143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:57.510226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:57.510407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518550681273328:2050] [node 3] ICS01 socket: 213 reason# EndOfStream
2025-11-11T17:18:57.510439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518546386305944:2050] [node 3] ICP30 unregister session Session# [4:7571518550681273328:2050] VirtualId# [4:7571518550681273319:0]
2025-11-11T17:18:57.510461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518550681273328:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.510511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:57.510540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:57.565762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:57.565762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:57.566119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518551330626550:2068] [node 4] ICS01 socket: 239 reason# EndOfStream
2025-11-11T17:18:57.566182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518547035659219:2068] [node 4] ICP30 unregister session Session# [5:7571518551330626550:2068] VirtualId# [5:7571518551330626548:0]
2025-11-11T17:18:57.566243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518551330626550:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.566383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:57.566468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:57.567068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518548724090393:2067] [node 4] ICS01 socket: 204 reason# EndOfStream
2025-11-11T17:18:57.567147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518544429123067:2067] [node 4] ICP30 unregister session Session# [6:7571518548724090393:2067] VirtualId# [6:7571518548724090382:0]
2025-11-11T17:18:57.567190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518548724090393:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:57.567287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:57.567348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:18:57.880028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518548682100927:2048] [node 2] ICP01 ready to work
2025-11-11T17:18:57.880071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518548682100928:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:57.930868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518548735245030:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:57.930912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518548735245031:2049] [node 3] ICP01 ready to work
2025-11-11T17:18:58.017629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518548441768686:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:58.017701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518548441768687:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:58.061871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518554034364950:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:58.061927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518554034364951:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:58.061955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518554034364952:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:58.094134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518552772686487:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:58.094173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518552772686488:2049] [node 2] ICP01 ready to work
2025-11-11T17:18:58.094198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518552772686489:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:58.206304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518553030212339:2059] [node 6] ICP01 ready to work
2025-11-11T17:18:58.257658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518552772686507:2060] [node 4] ICP01 ready to work
2025-11-11T17:18:58.237777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518552977068242:2060] [node 4] ICP01 ready to work
2025-11-11T17:18:58.269857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518554034364968:2062] [node 6] ICP01 ready to work
2025-11-11T17:18:58.277892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518552977068244:2062] [node 5] ICP01 ready to work
2025-11-11T17:18:58.379842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518554034364968:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(108085363ns) {Configure:104(108082014ns) {ProcessConfigured:135(85921082ns) {StartInitialHandshake:157(85918147ns) {DropHandshakes:469(3484ns) {DropIncomingHandshake:441(778ns) DropOutgoingHandshake:456(570ns)} GenerateSessionVirtualId:988(469ns) PrepareNewSessionHandshake:476(4822ns) {SwitchToState:178(1980ns)}}}}}
2025-11-11T17:18:58.380406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518554034364980:2069] [node 5] ICP01 ready to work
2025-11-11T17:18:58.304415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518552977068245:2064] [node 6] ICP01 ready to work
2025-11-11T17:18:58.434476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518548441768686:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(225523733ns) {Configure:104(225516163ns) {ProcessConfigured:135(62952477ns) {StartInitialHandshake:157(24636495ns) {DropHandshakes:469(5156ns) {DropIncomingHandshake:441(1128ns) DropOutgoingHandshake:456(707ns)} GenerateSessionVirtualId:988(3627ns) PrepareNewSessionHandshake:476(5211ns) {SwitchToState:178(2552ns)}}}}}
2025-11-11T17:18:58.495434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518552977068244:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(189381954ns) {Configure:104(161349476ns) {ProcessConfigured:135(77420ns) {StartInitialHandshake:157(74258ns) {DropHandshakes:469(3792ns) {DropIncomingHandshake:441(958ns) DropOutgoingHandshake:456(702ns)} GenerateSessionVirtualId:988(825ns) PrepareNewSessionHandshake:476(1876ns) {SwitchToState:178(650ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.522544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518548735245031:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(315408128ns) {Configure:104(315401911ns) {ProcessConfigured:135(69556ns) {StartInitialHandshake:157(66852ns) {DropHandshakes:469(1655ns) {DropIncomingHandshake:441(427ns) DropOutgoingHandshake:456(311ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:476(2032ns) {SwitchToState:178(812ns)}}}}}
2025-11-11T17:18:58.516928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518552736736028:2065] [node 4] ICP01 ready to work
2025-11-11T17:18:58.517604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518552736736023:2062] [node 6] ICP01 ready to work
2025-11-11T17:18:58.567469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518552736736045:2088] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.606586Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518552977068245:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(109180210ns) {Configure:104(109176530ns) {ProcessConfigured:135(65548ns) {StartInitialHandshake:157(62512ns) {DropHandshakes:469(2931ns) {DropIncomingHandshake:441(784ns) DropOutgoingHandshake:456(637ns)} GenerateSessionVirtualId:988(1113ns) PrepareNewSessionHandshake:476(1564ns) {SwitchToState:178(534ns)}}}}}
2025-11-11T17:18:58.608654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518552977068290:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16466
2025-11-11T17:18:58.609089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518552977068296:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16466
2025-11-11T17:18:58.609224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518552977068297:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16466
2025-11-11T17:18:58.523065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518553030212390:2067] [node 5] ICP01 ready to work
2025-11-11T17:18:58.365573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518554529924218:2048] [node 1] ICP01 ready to work
2025-11-11T17:18:58.570066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518554034364950:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(189246120ns) {Configure:104(189241219ns) {ProcessConfigured:135(128115315ns) {StartInitialHandshake:157(128112308ns) {DropHandshakes:469(3324ns) {DropIncomingHandshake:441(821ns) DropOutgoingHandshake:456(509ns)} GenerateSessionVirtualId:988(887ns) PrepareNewSessionHandshake:476(2327ns) {SwitchToState:178(1128ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.572143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518554034365016:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
2025-11-11T17:18:58.523380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518553030212391:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:58.636085Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068296:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.365666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518554529924219:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.685883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518554034364951:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(435948678ns) {Configure:104(435943555ns) {ProcessConfigured:135(45636ns) {StartInitialHandshake:157(43539ns) {DropHandshakes:469(1338ns) {DropIncomingHandshake:441(324ns) DropOutgoingHandshake:456(254ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:476(838ns) {SwitchToState:178(389ns)}}}}}
2025-11-11T17:18:58.586458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518553030212339:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(379838288ns) {Configure:104(379832957ns) {ProcessConfigured:135(28581778ns) {StartInitialHandshake:157(28578141ns) {DropHandshakes:469(3119ns) {DropIncomingHandshake:441(738ns) DropOutgoingHandshake:456(715ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:476(4266ns) {SwitchToState:178(1664ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.646003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518552772686489:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(380174635ns) {Configure:104(380162510ns) {ProcessConfigured:135(56924ns) {StartInitialHandshake:157(54069ns) {DropHandshakes:469(3764ns) {DropIncomingHandshake:441(913ns) DropOutgoingHandshake:456(597ns)} GenerateSessionVirtualId:988(681ns) PrepareNewSessionHandshake:476(2290ns) {SwitchToState:178(661ns)}}}}}
2025-11-11T17:18:58.665636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068297:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.714009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518552736736023:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(117174798ns) {Configure:104(117164873ns) {ProcessConfigured:135(62327ns) {StartInitialHandshake:157(58890ns) {DropHandshakes:469(3734ns) {DropIncomingHandshake:441(838ns) DropOutgoingHandshake:456(656ns)} GenerateSessionVirtualId:988(824ns) PrepareNewSessionHandshake:476(1686ns) {SwitchToState:178(615ns)}}}}}
2025-11-11T17:18:58.716456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518552736736056:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24357
2025-11-11T17:18:58.716596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518552736736057:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24357
2025-11-11T17:18:58.716674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518552736736058:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24357
2025-11-11T17:18:58.716747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518552736736059:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24357
2025-11-11T17:18:58.365703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518554529924220:2050] [node 3] ICP01 ready to work
2025-11-11T17:18:58.588159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518553030212392:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18784
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.647623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518552772686555:2075] [node 6] ICP01 ready to work
2025-11-11T17:18:58.697977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518554034365029:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.366303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518554529924242:2070] [node 4] ICP01 ready to work
2025-11-11T17:18:58.598312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518553030212390:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(73086732ns) {Configure:104(73083381ns) {ProcessConfigured:135(62877ns) {StartInitialHandshake:157(60137ns) {DropHandshakes:469(3565ns) {DropIncomingHandshake:441(787ns) DropOutgoingHandshake:456(680ns)} GenerateSessionVirtualId:988(707ns) PrepareNewSessionHandshake:476(1629ns) {SwitchToState:178(685ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.674165Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068290:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.709632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518554034365016:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.678757Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518552772686487:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16703316ns) {Configure:104(16696325ns) {ProcessConfigured:135(58171ns) {StartInitialHandshake:157(55622ns) {DropHandshakes:469(3341ns) {DropIncomingHandshake:441(785ns) DropOutgoingHandshake:456(611ns)} GenerateSessionVirtualId:988(978ns) PrepareNewSessionHandshake:476(1675ns) {SwitchToState:178(705ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.733881Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736057:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.694597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518552977068303:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16466
2025-11-11T17:18:58.648020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212392:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.738760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518554034364968:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(16179722ns) {IncomingHandshake:261(7979964ns) {DropIncomingHandshake:441(542ns) PrepareNewSessionHandshake:476(780ns) {SwitchToState:178(366ns)} GenerateSessionVirtualId:988(395ns) IssueIncomingHandshakeReply:193(6099ns)}}
2025-11-11T17:18:58.680309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518552772686556:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:58.734054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736056:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.555659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518554529924220:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(186584213ns) {Configure:104(186581274ns) {ProcessConfigured:135(37780910ns) {StartInitialHandshake:157(37778296ns) {DropHandshakes:469(3021ns) {DropIncomingHandshake:441(719ns) DropOutgoingHandshake:456(555ns)} GenerateSessionVirtualId:988(695ns) PrepareNewSessionHandshake:476(3423ns) {SwitchToState:178(1351ns)}}}}}
2025-11-11T17:18:58.705995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518552772686488:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(451354129ns) {Configure:104(451349820ns) {ProcessConfigured:135(12032550ns) {StartInitialHandshake:157(12030310ns) {DropHandshakes:469(2011ns) {DropIncomingHandshake:441(550ns) DropOutgoingHandshake:456(210ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:476(3044ns) {SwitchToState:178(1144ns)}}}}}
2025-11-11T17:18:58.727686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518552772686556:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.745491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518552772686487:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.766371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518552772686567:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:58.790627Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518552772686556:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:58.790830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518552772686487:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:58.790886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:58.790925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518552772686487:2048] [node 1] ICP22 created new session: [5:7571518552772686568:2048]
2025-11-11T17:18:58.790962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518552772686568:2048] [node 1] ICS09 handshake done sender: [5:7571518552772686556:2112] self: [5:7571518552772686566:0] peer: [1:7571518552977068286:0] socket: 212 qp: -1
2025-11-11T17:18:58.790996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518552772686568:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:58.791029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:58.705902Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212398:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.728693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068295:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.762015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518554034364979:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.762047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518554034364979:2074] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:58.797646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518554034365029:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.799238Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518554034364979:2074] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.601904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518554529924287:2077] [node 5] ICP01 ready to work
2025-11-11T17:18:58.705928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518553030212398:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:58.734132Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736058:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.728716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518552977068295:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.880847Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518552772686560:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.880879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518552772686560:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.717268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518553030212407:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18784
2025-11-11T17:18:58.635345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924296:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.753793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518548441768687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.753869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518548441768686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.774288Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736059:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.774667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518552736736065:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24357
2025-11-11T17:18:58.782274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518552736736028:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(192460342ns) {Configure:104(192450974ns) {ProcessConfigured:135(70967ns) {StartInitialHandshake:157(69174ns) {DropHandshakes:469(3315ns) {DropIncomingHandshake:441(963ns) DropOutgoingHandshake:456(663ns)} GenerateSessionVirtualId:988(903ns) PrepareNewSessionHandshake:476(2216ns) {SwitchToState:178(498ns)}}}}}
2025-11-11T17:18:58.733959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518552977068242:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(401342756ns) {Configure:104(401334054ns) {ProcessConfigured:135(60521ns) {StartInitialHandshake:157(54670ns) {DropHandshakes:469(3286ns) {DropIncomingHandshake:441(718ns) DropOutgoingHandshake:456(626ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:476(2148ns) {SwitchToState:178(1109ns)}}}}}
2025-11-11T17:18:58.738576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518552977068306:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16466
2025-11-11T17:18:58.758020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068287:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.758044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518552977068287:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:58.758694Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068289:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.758711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518552977068289:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:18:58.773751Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518552977068295:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:58.781730Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518552977068287:2108] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:58.798114Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518552977068289:2109] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:58.798408Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068306:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.798475Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068303:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.798513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518552977068245:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:58.798573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:58.798612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518552977068245:2064] [node 6] ICP22 created new session: [1:7571518552977068313:2064]
2025-11-11T17:18:58.798652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518552977068313:2064] [node 6] ICS09 handshake done sender: [1:7571518552977068295:2113] self: [1:7571518552977068294:0] peer: [6:7571518554529924302:0] socket: 209 qp: -1
2025-11-11T17:18:58.798685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518552977068313:2064] [node 6] ICS10 traffic start
2025-11-11T17:18:58.798716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:58.834235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518552977068244:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:58.834285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:58.834336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518552977068244:2062] [node 5] ICP22 created new session: [1:7571518552977068315:2062]
2025-11-11T17:18:58.834378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518552977068315:2062] [node 5] ICS09 handshake done sender: [1:7571518552977068287:2108] self: [1:7571518552977068286:0] peer: [5:7571518552772686566:0] socket: 207 qp: -1
2025-11-11T17:18:58.834415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518552977068315:2062] [node 5] ICS10 traffic start
2025-11-11T17:18:58.834455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:58.835181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518548682100928:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:58.835214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:58.835253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518548682100928:2049] [node 3] ICP22 created new session: [1:7571518552977068317:2049]
2025-11-11T17:18:58.835294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518552977068317:2049] [node 3] ICS09 handshake done sender: [1:7571518552977068289:2109] self: [1:7571518552977068288:0] peer: [3:7571518552736736064:0] socket: 208 qp: -1
2025-11-11T17:18:58.835327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518552977068317:2049] [node 3] ICS10 traffic start
2025-11-11T17:18:58.835355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:58.635445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924297:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.736173Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518553030212398:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:58.635522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924298:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.657668Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518554529924297:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.673748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518554529924219:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.674119Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518554529924296:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.717743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518554529924219:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16006560ns) {IncomingHandshake:261(16004544ns) {DropIncomingHandshake:441(527ns) PrepareNewSessionHandshake:476(1858ns) {SwitchToState:178(815ns)} GenerateSessionVirtualId:988(765ns) IssueIncomingHandshakeReply:193(8403689ns) {SwitchToState:178(2324ns)}}}
2025-11-11T17:18:58.721813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518554529924298:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.721970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518554529924242:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.722197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518554529924218:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.722271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924300:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.760383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924304:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.761776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924303:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.808294Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518554529924300:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.808768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518554529924297:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:58.836190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518554529924219:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:58.836233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:58.836267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518554529924219:2049] [node 2] ICP22 created new session: [6:7571518554529924307:2049]
2025-11-11T17:18:58.836307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518554529924307:2049] [node 2] ICS09 handshake done sender: [6:7571518554529924297:2116] self: [6:7571518554529924299:0] peer: [2:7571518553030212397:0] socket: 210 qp: -1
2025-11-11T17:18:58.836339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518554529924307:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:58.836364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:58.836753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518554529924220:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.836854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924305:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.837154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924306:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.837273Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518554529924298:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:58.838195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518554529924218:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:58.838228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:58.838261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518554529924218:2048] [node 1] ICP22 created new session: [6:7571518554529924310:2048]
2025-11-11T17:18:58.838299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518554529924310:2048] [node 1] ICS09 handshake done sender: [6:7571518554529924298:2117] self: [6:7571518554529924302:0] peer: [1:7571518552977068294:0] socket: 211 qp: -1
2025-11-11T17:18:58.838325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518554529924310:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:58.838349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:58.838667Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:58.854408Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518554529924296:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:58.868599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518554529924218:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:18:58.868985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518554529924306:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.877983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518554529924242:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:58.878041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:58.878085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518554529924242:2070] [node 4] ICP22 created new session: [6:7571518554529924312:2070]
2025-11-11T17:18:58.878137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518554529924312:2070] [node 4] ICS09 handshake done sender: [6:7571518554529924296:2115] self: [6:7571518554529924301:0] peer: [4:7571518554034364975:0] socket: 204 qp: -1
2025-11-11T17:18:58.878175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518554529924312:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:58.878208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:58.880178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518554529924287:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:58.888048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924317:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.914124Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518554529924306:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:58.893789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518552772686560:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:58.901978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518552772686571:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:58.942206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518554034364968:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:58.914208Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736053:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.914257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518552736736053:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:18:58.930717Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518552736736058:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.953857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518552977068244:2062] [node 5] ICP18 handshake (actor [1:7571518552977068303:2117]) is too old
2025-11-11T17:18:58.833641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212389:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.954129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518554529924287:2077] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:18:58.954192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:58.954237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518554529924287:2077] [node 5] ICP22 created new session: [6:7571518554529924318:2077]
2025-11-11T17:18:58.954282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518554529924318:2077] [node 5] ICS09 handshake done sender: [6:7571518554529924306:2122] self: [6:7571518554529924316:0] peer: [5:7571518552772686559:0] socket: 238 qp: -1
2025-11-11T17:18:58.954316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518554529924318:2077] [node 5] ICS10 traffic start
2025-11-11T17:18:58.954351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.942283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:58.949120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518552772686555:2075] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.833692Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518553030212389:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.846593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518553030212339:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:58.846644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:58.846681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518553030212339:2059] [node 6] ICP22 created new session: [2:7571518553030212409:2059]
2025-11-11T17:18:58.846724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518553030212409:2059] [node 6] ICS09 handshake done sender: [2:7571518553030212398:2113] self: [2:7571518553030212397:0] peer: [6:7571518554529924299:0] socket: 203 qp: -1
2025-11-11T17:18:58.846755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518553030212409:2059] [node 6] ICS10 traffic start
2025-11-11T17:18:58.846790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:58.847286Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212407:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.921157Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518553030212389:2108] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:58.921641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518548735245031:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:18:58.950469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:58.954902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518548735245031:2049] [node 3] ICP22 created new session: [2:7571518553030212415:2049]
2025-11-11T17:18:58.974619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518553030212415:2049] [node 3] ICS09 handshake done sender: [2:7571518553030212389:2108] self: [2:7571518553030212388:0] peer: [3:7571518552736736063:0] socket: 201 qp: -1
2025-11-11T17:18:58.982559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518553030212415:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.961895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518552736736071:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24357
2025-11-11T17:18:58.961725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518548682100928:2049] [node 3] ICP18 handshake (actor [1:7571518552977068306:2118]) is too old
2025-11-11T17:18:58.978463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518554529924320:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53851
2025-11-11T17:18:58.949171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:58.942328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518554034364968:2062] [node 6] ICP22 created new session: [4:7571518554034365034:2062]
2025-11-11T17:18:59.003610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.978644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518552736736070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24357
2025-11-11T17:18:58.942368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518554034365034:2062] [node 6] ICS09 handshake done sender: [4:7571518554034364979:2074] self: [4:7571518554034364975:0] peer: [6:7571518554529924301:0] socket: 199 qp: -1
2025-11-11T17:18:58.949215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518552772686555:2075] [node 6] ICP22 created new session: [5:7571518552772686574:2075]
2025-11-11T17:18:58.962158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518552977068245:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(89858793ns) {HandleHandshakeStatus:329(89855612ns) {DropIncomingHandshake:441(13722ns) DropOutgoingHandshake:456(640ns) SwitchToState:178(848ns) ProcessPendingSessionEvents:495(46560108ns) {ForwardSessionEventToSession:612(21726629ns) ForwardSessionEventToSession:612(6893ns) ForwardSessionEventToSession:612(2998ns) ForwardSessionEventToSession:612(19949495ns) ForwardSessionEventToSession:612(15741ns) ForwardSessionEventToSession:612(3696ns) ForwardSessionEventToSession:612(357278ns) ForwardSessionEventToSession:612(3329ns) ForwardSessionEventToSession:612(4470371ns) ForwardSessionEventToSession:612(4004ns)}}}
2025-11-11T17:18:58.949257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518552772686574:2075] [node 6] ICS09 handshake done sender: [5:7571518552772686560:2114] self: [5:7571518552772686559:0] peer: [6:7571518554529924316:0] socket: 215 qp: -1
2025-11-11T17:18:58.949297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518552772686574:2075] [node 6] ICS10 traffic start
2025-11-11T17:18:58.949331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:58.950066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518552772686573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:58.969859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518552772686571:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.970203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518552772686555:2075] [node 6] ICP18 handshake (actor [5:7571518552772686571:2117]) is too old
2025-11-11T17:18:59.003211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518557067653873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:59.023695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518553030212416:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18784
2025-11-11T17:18:58.999785Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:58.942404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518554034365034:2062] [node 6] ICS10 traffic start
2025-11-11T17:18:59.057179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518557067653873:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.057828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518552772686573:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.057956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518552772686488:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:59.027961Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518554529924300:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:59.023822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518553030212417:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18784
2025-11-11T17:18:59.066706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518552772686507:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:58.942440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.001756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518548441768686:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.056881Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068299:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:58.965317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518554034365033:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
2025-11-11T17:18:58.982352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518554034365037:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
2025-11-11T17:18:59.010109Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518554034365033:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.026561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518554034364951:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:59.053713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518554034365037:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.073356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518558329332336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
2025-11-11T17:18:59.073806Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518554034365033:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:59.073995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518554034364951:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.074067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:59.074114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518554034364951:2049] [node 2] ICP22 created new session: [4:7571518558329332337:2049]
2025-11-11T17:18:59.074174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518558329332337:2049] [node 2] ICS09 handshake done sender: [4:7571518554034365033:2117] self: [4:7571518558329332334:0] peer: [2:7571518553030212399:0] socket: 241 qp: -1
2025-11-11T17:18:59.074211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518558329332337:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:59.074245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:59.078217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518554034364950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:59.065711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518554529924220:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.054438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518548735245031:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(132638246ns) {HandleHandshakeStatus:329(132633328ns) {DropIncomingHandshake:441(15941ns) DropOutgoingHandshake:456(541ns) SwitchToState:178(656ns) ProcessPendingSessionEvents:495(19865441ns) {ForwardSessionEventToSession:612(19802733ns) ForwardSessionEventToSession:612(8060ns) ForwardSessionEventToSession:612(7340ns) ForwardSessionEventToSession:612(5943ns) ForwardSessionEventToSession:612(2902ns) ForwardSessionEventToSession:612(5583ns) ForwardSessionEventToSession:612(3121ns) ForwardSessionEventToSession:612(2690ns) ForwardSessionEventToSession:612(8466ns) ForwardSessionEventToSession:612(2478ns)}}}
2025-11-11T17:18:59.056920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518552977068299:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:18:59.078513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518557067653876:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:59.001826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:59.085702Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518554034365013:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.085737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518554034365013:2105] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:59.086783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518558329332339:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
2025-11-11T17:18:59.087415Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518554034365013:2105] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:59.087619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518554034364980:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.087662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:59.087698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518554034364980:2069] [node 5] ICP22 created new session: [4:7571518558329332341:2069]
2025-11-11T17:18:59.087751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518558329332341:2069] [node 5] ICS09 handshake done sender: [4:7571518554034365013:2105] self: [4:7571518554034365012:0] peer: [5:7571518557067653875:0] socket: 250 qp: -1
2025-11-11T17:18:59.087787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518558329332341:2069] [node 5] ICS10 traffic start
2025-11-11T17:18:59.087815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:59.088171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518554034365037:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:59.088677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518554034364950:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.088712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:59.088748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518554034364950:2048] [node 1] ICP22 created new session: [4:7571518558329332343:2048]
2025-11-11T17:18:59.088786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518558329332343:2048] [node 1] ICS09 handshake done sender: [4:7571518554034365037:2118] self: [4:7571518558329332335:0] peer: [1:7571518552977068307:0] socket: 193 qp: -1
2025-11-11T17:18:59.088815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518558329332343:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:59.088840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:59.078302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.079407Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518552977068308:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.001874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518548441768686:2048] [node 1] ICP22 created new session: [3:7571518557031703369:2048]
2025-11-11T17:18:59.054601Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212400:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.080440Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518552772686573:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.001915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518557031703369:2048] [node 1] ICS09 handshake done sender: [3:7571518552736736058:2115] self: [3:7571518552736736064:0] peer: [1:7571518552977068288:0] socket: 220 qp: -1
2025-11-11T17:18:59.001952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518557031703369:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:59.002017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:59.002684Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518552736736053:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:18:59.028295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518548441768687:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518552736736057:2114] held: no
2025-11-11T17:18:59.028461Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736070:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.065866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518552736736023:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.065911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:18:59.065962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518552736736023:2062] [node 6] ICP22 created new session: [3:7571518557031703371:2062]
2025-11-11T17:18:59.066006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518557031703371:2062] [node 6] ICS09 handshake done sender: [3:7571518552736736053:2112] self: [3:7571518552736736052:0] peer: [6:7571518554529924309:0] socket: 224 qp: -1
2025-11-11T17:18:59.066043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518557031703371:2062] [node 6] ICS10 traffic start
2025-11-11T17:18:59.066099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:18:59.066539Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518552736736057:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:59.114869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518548441768687:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.114920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:59.114955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518548441768687:2049] [node 2] ICP22 created new session: [3:7571518557031703373:2049]
2025-11-11T17:18:59.114990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518557031703373:2049] [node 2] ICS09 handshake done sender: [3:7571518552736736057:2114] self: [3:7571518552736736063:0] peer: [2:7571518553030212388:0] socket: 219 qp: -1
2025-11-11T17:18:59.115033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518557031703373:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:59.115068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:59.137726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518552736736023:2062] [node 6] ICP18 handshake (actor [3:7571518552736736070:2120]) is too old
2025-11-11T17:18:59.079430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518552977068308:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:59.080722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518554529924220:2050] [node 3] ICP22 created new session: [6:7571518558824891617:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.082746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518552772686488:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.054624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518553030212400:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:18:59.080776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518558824891617:2050] [node 3] ICS09 handshake done sender: [6:7571518554529924300:2118] self: [6:7571518554529924309:0] peer: [3:7571518552736736052:0] socket: 226 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.182100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518558329332345:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
2025-11-11T17:18:59.090052Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518552977068308:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:59.108368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518552977068242:2060] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.108430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:59.108476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518552977068242:2060] [node 4] ICP22 created new session: [1:7571518557272035619:2060]
2025-11-11T17:18:59.108518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518557272035619:2060] [node 4] ICS09 handshake done sender: [1:7571518552977068308:2119] self: [1:7571518552977068307:0] peer: [4:7571518558329332335:0] socket: 248 qp: -1
2025-11-11T17:18:59.108552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518557272035619:2060] [node 4] ICS10 traffic start
2025-11-11T17:18:59.108589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:59.127396Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518552977068299:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:18:59.153312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518548682100927:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.153369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:59.153411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518548682100927:2048] [node 2] ICP22 created new session: [1:7571518557272035621:2048]
2025-11-11T17:18:59.153459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518557272035621:2048] [node 2] ICS09 handshake done sender: [1:7571518552977068299:2116] self: [1:7571518552977068298:0] peer: [2:7571518557325179716:0] socket: 197 qp: -1
2025-11-11T17:18:59.153487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518557272035621:2048] [node 2] ICS10 traffic start
2025-11-11T17:18:59.153521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:59.080811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518558824891617:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:59.080849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:59.091580Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17152) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:18:59.109517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518552772686507:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17152) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518557067653873:2119] held: no
2025-11-11T17:18:59.113670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:18:59.113741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518552772686488:2049] [node 2] ICP22 created new session: [5:7571518557067653878:2049]
2025-11-11T17:18:59.113792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518557067653878:2049] [node 2] ICS09 handshake done sender: [5:7571518552772686573:2118] self: [5:7571518557067653874:0] peer: [2:7571518553030212404:0] socket: 244 qp: -1
2025-11-11T17:18:59.113828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518557067653878:2049] [node 2] ICS10 traffic start
2025-11-11T17:18:59.113883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:18:59.114302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518552772686488:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31387405ns) {HandleHandshakeStatus:329(31384668ns) {DropIncomingHandshake:441(722ns) DropOutgoingHandshake:456(4618ns) SwitchToState:178(544ns) ProcessPendingSessionEvents:495(106501ns) {ForwardSessionEventToSession:612(57109ns) ForwardSessionEventToSession:612(5648ns) ForwardSessionEventToSession:612(8428ns) ForwardSessionEventToSession:612(3067ns) ForwardSessionEventToSession:612(5575ns) ForwardSessionEventToSession:612(3158ns) ForwardSessionEventToSession:612(2866ns) ForwardSessionEventToSession:612(4746ns) ForwardSessionEventToSession:612(1869ns) ForwardSessionEventToSession:612(5082ns)}}}
2025-11-11T17:18:59.122072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518557067653877:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:59.122860Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518557067653873:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:59.123015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518552772686507:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.123058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:59.123097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518552772686507:2060] [node 4] ICP22 created new session: [5:7571518557067653880:2060]
2025-11-11T17:18:59.123148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518557067653880:2060] [node 4] ICS09 handshake done sender: [5:7571518557067653873:2119] self: [5:7571518557067653875:0] peer: [4:7571518554034365012:0] socket: 251 qp: -1
2025-11-11T17:18:59.123179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518557067653880:2060] [node 4] ICS10 traffic start
2025-11-11T17:18:59.123209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:59.152939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518557067653882:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:59.182407Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518557067653882:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.182616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518552772686489:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:59.055488Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212417:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.056170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518548735245030:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:59.061819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212416:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.066816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518553030212400:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:59.071585Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518553030212405:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.071614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518553030212405:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:59.077837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518553030212405:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:18:59.131837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518553030212391:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.131881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:59.131931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518553030212391:2070] [node 4] ICP22 created new session: [2:7571518557325179719:2070]
2025-11-11T17:18:59.131974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518557325179719:2070] [node 4] ICS09 handshake done sender: [2:7571518553030212400:2114] self: [2:7571518553030212399:0] peer: [4:7571518558329332334:0] socket: 240 qp: -1
2025-11-11T17:18:59.132009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518557325179719:2070] [node 4] ICS10 traffic start
2025-11-11T17:18:59.132043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:59.132631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518553030212390:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.132671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:59.132703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518553030212390:2067] [node 5] ICP22 created new session: [2:7571518557325179721:2067]
2025-11-11T17:18:59.132733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518557325179721:2067] [node 5] ICS09 handshake done sender: [2:7571518553030212405:2115] self: [2:7571518553030212404:0] peer: [5:7571518557067653874:0] socket: 225 qp: -1
2025-11-11T17:18:59.132755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518557325179721:2067] [node 5] ICS10 traffic start
2025-11-11T17:18:59.132775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:59.142285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518557325179723:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18784
2025-11-11T17:18:59.152265Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518553030212417:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:18:59.154331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518548735245030:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.154371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:18:59.154404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518548735245030:2048] [node 1] ICP22 created new session: [2:7571518557325179724:2048]
2025-11-11T17:18:59.154444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518557325179724:2048] [node 1] ICS09 handshake done sender: [2:7571518553030212417:2118] self: [2:7571518557325179716:0] peer: [1:7571518552977068298:0] socket: 194 qp: -1
2025-11-11T17:18:59.154466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518557325179724:2048] [node 1] ICS10 traffic start
2025-11-11T17:18:59.154494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:18:59.191074Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736061:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.191116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518552736736061:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:18:59.192109Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518558329332345:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.192251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518554034364952:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:18:59.197856Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518552736736061:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.198574Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518552736736068:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:18:59.198603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518552736736068:2119] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:18:59.200045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518558329332347:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:67
2025-11-11T17:18:59.198306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518557067653884:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59129
2025-11-11T17:18:59.202024Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518557067653882:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:59.202278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518552772686489:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.202340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:59.202380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518552772686489:2050] [node 3] ICP22 created new session: [5:7571518557067653885:2050]
2025-11-11T17:18:59.202422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518557067653885:2050] [node 3] ICS09 handshake done sender: [5:7571518557067653882:2122] self: [5:7571518557067653883:0] peer: [3:7571518552736736060:0] socket: 254 qp: -1
2025-11-11T17:18:59.202454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518557067653885:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:59.202490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:59.206106Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518552736736068:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:18:59.206714Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518558329332345:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:18:59.206960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518552736736045:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.207048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:18:59.207095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518552736736045:2088] [node 5] ICP22 created new session: [3:7571518557031703379:2088]
2025-11-11T17:18:59.207135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518557031703379:2088] [node 5] ICS09 handshake done sender: [3:7571518552736736061:2117] self: [3:7571518552736736060:0] peer: [5:7571518557067653883:0] socket: 205 qp: -1
2025-11-11T17:18:59.207172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518557031703379:2088] [node 5] ICS10 traffic start
2025-11-11T17:18:59.207241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:18:59.207886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518552736736028:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:18:59.207946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:18:59.208020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518552736736028:2065] [node 4] ICP22 created new session: [3:7571518557031703381:2065]
2025-11-11T17:18:59.208054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518557031703381:2065] [node 4] ICS09 handshake done sender: [3:7571518552736736068:2119] self: [3:7571518552736736067:0] peer: [4:7571518558329332346:0] socket: 214 qp: -1
2025-11-11T17:18:59.208079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518557031703381:2065] [node 4] ICS10 traffic start
2025-11-11T17:18:59.208100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:18:59.219165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518554034364952:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:18:59.219250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:18:59.219302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518554034364952:2050] [node 3] ICP22 created new session: [4:7571518558329332348:2050]
2025-11-11T17:18:59.219349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518558329332348:2050] [node 3] ICS09 handshake done sender: [4:7571518558329332345:2121] self: [4:7571518558329332346:0] peer: [3:7571518552736736067:0] socket: 202 qp: -1
2025-11-11T17:18:59.219382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518558329332348:2050] [node 3] ICS10 traffic start
2025-11-11T17:18:59.219419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:18:59.313778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:18:59.314035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518557325179724:2048] [node 1] ICS01 socket: 194 reason# EndOfStream
2025-11-11T17:18:59.314069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518548735245030:2048] [node 1] ICP30 unregister session Session# [2:7571518557325179724:2048] VirtualId# [2:7571518557325179716:0]
2025-11-11T17:18:59.314102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518557325179724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.314201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:59.314244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:59.314321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:18:59.314531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518557031703369:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-11T17:18:59.314560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518548441768686:2048] [node 1] ICP30 unregister session Session# [3:7571518557031703369:2048] VirtualId# [3:7571518552736736064:0]
2025-11-11T17:18:59.314583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518557031703369:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.314628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:59.314655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:59.321191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:18:59.321438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518552772686568:2048] [node 1] ICS01 socket: 212 reason# EndOfStream
2025-11-11T17:18:59.321477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518552772686487:2048] [node 1] ICP30 unregister session Session# [5:7571518552772686568:2048] VirtualId# [5:7571518552772686566:0]
2025-11-11T17:18:59.321507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518552772686568:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.321631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:59.321679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:59.321726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:18:59.321965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518558329332343:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-11-11T17:18:59.321998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518554034364950:2048] [node 1] ICP30 unregister session Session# [4:7571518558329332343:2048] VirtualId# [4:7571518558329332335:0]
2025-11-11T17:18:59.322024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518558329332343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.322088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:59.322127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:59.333760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:18:59.333996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518554529924310:2048] [node 1] ICS01 socket: 211 reason# EndOfStream
2025-11-11T17:18:59.334043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518554529924218:2048] [node 1] ICP30 unregister session Session# [6:7571518554529924310:2048] VirtualId# [6:7571518554529924302:0]
2025-11-11T17:18:59.334080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518554529924310:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.334169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:18:59.334213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:18:59.405235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:18:59.405481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518557031703373:2049] [node 2] ICS01 socket: 219 reason# EndOfStream
2025-11-11T17:18:59.405515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518548441768687:2049] [node 2] ICP30 unregister session Session# [3:7571518557031703373:2049] VirtualId# [3:7571518552736736063:0]
2025-11-11T17:18:59.405547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518557031703373:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.405629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:59.405666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:59.425792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:18:59.425865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:18:59.426023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518558329332337:2049] [node 2] ICS01 socket: 241 reason# EndOfStream
2025-11-11T17:18:59.426080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518557067653878:2049] [node 2] ICS01 socket: 244 reason# EndOfStream
2025-11-11T17:18:59.426065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518554034364951:2049] [node 2] ICP30 unregister session Session# [4:7571518558329332337:2049] VirtualId# [4:7571518558329332334:0]
2025-11-11T17:18:59.426115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518552772686488:2049] [node 2] ICP30 unregister session Session# [5:7571518557067653878:2049] VirtualId# [5:7571518557067653874:0]
2025-11-11T17:18:59.426102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518558329332337:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.426144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518557067653878:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.426193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:59.426222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:59.426240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:59.426256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:59.425792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:18:59.426786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518554529924307:2049] [node 2] ICS01 socket: 210 reason# EndOfStream
2025-11-11T17:18:59.426823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518554529924219:2049] [node 2] ICP30 unregister session Session# [6:7571518554529924307:2049] VirtualId# [6:7571518554529924299:0]
2025-11-11T17:18:59.426847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518554529924307:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.426902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:18:59.426936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:18:59.553114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:18:59.553114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:18:59.553371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518558329332348:2050] [node 3] ICS01 socket: 202 reason# EndOfStream
2025-11-11T17:18:59.553371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518558824891617:2050] [node 3] ICS01 socket: 226 reason# EndOfStream
2025-11-11T17:18:59.553406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518554034364952:2050] [node 3] ICP30 unregister session Session# [4:7571518558329332348:2050] VirtualId# [4:7571518558329332346:0]
2025-11-11T17:18:59.553438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518558329332348:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.553416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518554529924220:2050] [node 3] ICP30 unregister session Session# [6:7571518558824891617:2050] VirtualId# [6:7571518554529924309:0]
2025-11-11T17:18:59.553536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:59.553441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518558824891617:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.553611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:59.553538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:59.553579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:59.553935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:18:59.554150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518557067653885:2050] [node 3] ICS01 socket: 254 reason# EndOfStream
2025-11-11T17:18:59.554176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518552772686489:2050] [node 3] ICP30 unregister session Session# [5:7571518557067653885:2050] VirtualId# [5:7571518557067653883:0]
2025-11-11T17:18:59.554199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518557067653885:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.554270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:18:59.554311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:18:59.651914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:18:59.652169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518557067653880:2060] [node 4] ICS01 socket: 251 reason# EndOfStream
2025-11-11T17:18:59.652205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518552772686507:2060] [node 4] ICP30 unregister session Session# [5:7571518557067653880:2060] VirtualId# [5:7571518557067653875:0]
2025-11-11T17:18:59.652239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518557067653880:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.652315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:59.652371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:59.653748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:18:59.653957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518554529924312:2070] [node 4] ICS01 socket: 204 reason# EndOfStream
2025-11-11T17:18:59.653987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518554529924242:2070] [node 4] ICP30 unregister session Session# [6:7571518554529924312:2070] VirtualId# [6:7571518554529924301:0]
2025-11-11T17:18:59.654014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518554529924312:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.654093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:18:59.654134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:18:59.837718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:18:59.837986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518554529924318:2077] [node 5] ICS01 socket: 238 reason# EndOfStream
2025-11-11T17:18:59.838022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518554529924287:2077] [node 5] ICP30 unregister session Session# [6:7571518554529924318:2077] VirtualId# [6:7571518554529924316:0]
2025-11-11T17:18:59.838055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518554529924318:2077] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:18:59.838180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:18:59.838220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:00.005078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518559164483825:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:00.005120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518559164483826:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:00.065879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518562595483485:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:00.065920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518562595483486:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:00.110034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518564040535813:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:00.110094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518564040535814:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:00.197990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518564570127600:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:00.198036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518564570127601:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:00.198063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518564570127602:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:00.253916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518561549419392:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:00.253971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518561549419393:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:00.253998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518561549419394:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:00.329939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518561584109563:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:00.329979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518561584109564:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:00.330062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518561584109565:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:00.602614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518562595483505:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.586431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518563459451175:2098] [node 4] ICP01 ready to work
2025-11-11T17:19:00.606909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518564570127613:2059] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.591041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518564040535834:2068] [node 5] ICP01 ready to work
2025-11-11T17:19:00.662195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518561549419406:2061] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:00.667585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518563459451186:2100] [node 5] ICP01 ready to work
2025-11-11T17:19:00.675269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518561584109574:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:00.721894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518564040535873:2074] [node 6] ICP01 ready to work
2025-11-11T17:19:00.697890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518561549419408:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:00.763019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518562595483552:2069] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.722523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518564570127624:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:00.667873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518563459451187:2101] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.786431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518563459451198:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1312
2025-11-11T17:19:00.749831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518561549419413:2060] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.799297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518564040535878:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.770646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518562595483485:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(165563723ns) {Configure:104(165560398ns) {ProcessConfigured:135(41317385ns) {StartInitialHandshake:157(41315225ns) {DropHandshakes:469(24809620ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(1782ns)} GenerateSessionVirtualId:988(2305ns) PrepareNewSessionHandshake:476(2504ns) {SwitchToState:178(1493ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.766366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518561549419415:2067] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.850861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518564570127655:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:00.772714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518562595483558:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44648
2025-11-11T17:19:00.857772Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518563459451172:2097] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:00.766527Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518561549419408:2063] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:00.954806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518561584109563:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(244882659ns) {Configure:104(244877054ns) {ProcessConfigured:135(61147ns) {StartInitialHandshake:157(58917ns) {DropHandshakes:469(3440ns) {DropIncomingHandshake:441(776ns) DropOutgoingHandshake:456(707ns)} GenerateSessionVirtualId:988(1315ns) PrepareNewSessionHandshake:476(1762ns) {SwitchToState:178(639ns)}}}}}
2025-11-11T17:19:00.955247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518561584109631:2072] [node 4] ICP01 ready to work
2025-11-11T17:19:00.967183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518564040535813:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(107051323ns) {Configure:104(107044168ns) {ProcessConfigured:135(55695ns) {StartInitialHandshake:157(51044ns) {DropHandshakes:469(2692ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(488ns)} GenerateSessionVirtualId:988(1047ns) PrepareNewSessionHandshake:476(2030ns) {SwitchToState:178(911ns)}}}}}
2025-11-11T17:19:00.857809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518563459451172:2097] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:00.772855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518562595483559:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44648
2025-11-11T17:19:00.764493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518562595483549:2062] [node 4] ICP01 ready to work
2025-11-11T17:19:00.799215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518562595483558:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:00.856151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518562595483485:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:00.856607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518562595483564:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44648
2025-11-11T17:19:00.874902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518562595483566:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44648
2025-11-11T17:19:00.875797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518562595483558:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:00.875934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518562595483485:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:00.875983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:00.876023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518562595483485:2048] [node 1] ICP22 created new session: [2:7571518562595483567:2048]
2025-11-11T17:19:00.876063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518562595483567:2048] [node 1] ICS09 handshake done sender: [2:7571518562595483558:2113] self: [2:7571518562595483565:0] peer: [1:7571518563459451171:0] socket: 207 qp: -1
2025-11-11T17:19:00.876094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518562595483567:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:00.876127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:00.927414Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518562595483559:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:00.941796Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518562595483563:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:00.941823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518562595483563:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:00.942697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518562595483563:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:00.942894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518562595483549:2062] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:00.942944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:00.942990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518562595483549:2062] [node 4] ICP22 created new session: [2:7571518562595483572:2062]
2025-11-11T17:19:00.943032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518562595483572:2062] [node 4] ICS09 handshake done sender: [2:7571518562595483563:2115] self: [2:7571518562595483562:0] peer: [4:7571518564570127679:0] socket: 210 qp: -1
2025-11-11T17:19:00.943057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518562595483572:2062] [node 4] ICS10 traffic start
2025-11-11T17:19:00.943092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.851048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518564570127656:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:00.806771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518561549419393:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(37367414ns) {Configure:104(37363852ns) {ProcessConfigured:135(72461ns) {StartInitialHandshake:157(68830ns) {DropHandshakes:469(3385ns) {DropIncomingHandshake:441(675ns) DropOutgoingHandshake:456(707ns)} GenerateSessionVirtualId:988(1014ns) PrepareNewSessionHandshake:476(1923ns) {SwitchToState:178(670ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.983335Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518562595483564:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:00.914968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518564570127624:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(148884508ns) {Configure:104(141110628ns) {ProcessConfigured:135(96871ns) {StartInitialHandshake:157(94012ns) {DropHandshakes:469(3666ns) {DropIncomingHandshake:441(661ns) DropOutgoingHandshake:456(529ns)} GenerateSessionVirtualId:988(876ns) PrepareNewSessionHandshake:476(2411ns) {SwitchToState:178(716ns)}}}}}
2025-11-11T17:19:00.861662Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518563459451198:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.013774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518562595483507:2067] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.013818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518562595483507:2067] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:00.807655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518561549419430:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:00.969704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518561584109626:2070] [node 5] ICP01 ready to work
2025-11-11T17:19:01.026887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518562595483552:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(164978711ns) {Configure:104(164971413ns) {ProcessConfigured:135(70744ns) {StartInitialHandshake:157(67901ns) {DropHandshakes:469(4213ns) {DropIncomingHandshake:441(374ns) DropOutgoingHandshake:456(222ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:476(2272ns) {SwitchToState:178(1023ns)}}}}}
2025-11-11T17:19:00.998876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518564040535873:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(204958429ns) {Configure:104(204951264ns) {ProcessConfigured:135(73144ns) {StartInitialHandshake:157(70489ns) {DropHandshakes:469(3181ns) {DropIncomingHandshake:441(808ns) DropOutgoingHandshake:456(609ns)} GenerateSessionVirtualId:988(1244ns) PrepareNewSessionHandshake:476(2135ns) {SwitchToState:178(656ns)}}}}}
2025-11-11T17:19:00.999486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518564040535884:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26107
2025-11-11T17:19:00.875300Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518563459451172:2097] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:00.915205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518564570127655:2096] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.033949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518564040535814:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(319528151ns) {Configure:104(319521998ns) {ProcessConfigured:135(8051689ns) {StartInitialHandshake:157(8048346ns) {DropHandshakes:469(3272ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(651ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:476(1789ns) {SwitchToState:178(398ns)}}}}}
2025-11-11T17:19:01.049725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518564040535884:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.053756Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518562595483507:2067] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:00.894822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518561549419394:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(182436769ns) {Configure:104(182431855ns) {ProcessConfigured:135(63897ns) {StartInitialHandshake:157(60177ns) {DropHandshakes:469(5225ns) {DropIncomingHandshake:441(674ns) DropOutgoingHandshake:456(418ns)} GenerateSessionVirtualId:988(1103ns) PrepareNewSessionHandshake:476(1820ns) {SwitchToState:178(640ns)}}}}}
2025-11-11T17:19:01.060713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518564040535814:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:00.916044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518564570127676:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:00.877326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518559164483825:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.061375Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518562595483505:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(16273827ns) {IncomingHandshake:261(16270507ns) {DropIncomingHandshake:441(1068ns) PrepareNewSessionHandshake:476(6849944ns) {SwitchToState:178(3333ns)} GenerateSessionVirtualId:988(909ns) IssueIncomingHandshakeReply:193(8752ns)}}
2025-11-11T17:19:01.034429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518561584109618:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:00.960085Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518561549419430:2080] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.069720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076936:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:01.069794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076937:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:01.069924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076938:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.916139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518564570127600:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:00.877356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:01.063345Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518564040535836:2069] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.063352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518562595483505:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:00.973714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518561549419393:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:00.986518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518561549419392:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(328615286ns) {Configure:104(328608660ns) {ProcessConfigured:135(69197ns) {StartInitialHandshake:157(66921ns) {DropHandshakes:469(3801ns) {DropIncomingHandshake:441(1139ns) DropOutgoingHandshake:456(666ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:476(2314ns) {SwitchToState:178(692ns)}}}}}
2025-11-11T17:19:00.986704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518561549419392:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.038061Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518561549419406:2061] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.053891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518561549419394:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.060311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518565844386772:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.110822Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:01.111683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518561549419393:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518561549419430:2080] held: no
2025-11-11T17:19:01.112012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518565844386773:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:01.080180Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518561584109574:2058] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:00.877389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518559164483825:2048] [node 2] ICP22 created new session: [1:7571518563459451200:2048]
2025-11-11T17:19:00.929206Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518564570127676:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.063393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518564040535836:2069] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.114024Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518561549419430:2080] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:00.877435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518563459451200:2048] [node 2] ICS09 handshake done sender: [1:7571518563459451172:2097] self: [1:7571518563459451171:0] peer: [2:7571518562595483565:0] socket: 198 qp: -1
2025-11-11T17:19:00.877459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518563459451200:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:00.877482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:00.981840Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518563459451177:2099] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:00.981865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518563459451177:2099] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:00.982995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518563459451202:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1312
2025-11-11T17:19:01.001678Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518563459451177:2099] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:01.029712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518559164483825:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(64462565ns) {HandleHandshakeStatus:329(64460033ns) {DropIncomingHandshake:441(245ns) DropOutgoingHandshake:456(348ns) SwitchToState:178(117ns) ProcessPendingSessionEvents:495(64172355ns) {ForwardSessionEventToSession:612(4110074ns) ForwardSessionEventToSession:612(7176ns) ForwardSessionEventToSession:612(6242367ns) ForwardSessionEventToSession:612(8444ns) ForwardSessionEventToSession:612(21810926ns) ForwardSessionEventToSession:612(2969ns) ForwardSessionEventToSession:612(15959372ns) ForwardSessionEventToSession:612(4267ns) ForwardSessionEventToSession:612(3054ns) ForwardSessionEventToSession:612(16003590ns)}}}
2025-11-11T17:19:01.060614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518559164483825:2048] [node 2] ICP18 handshake (actor [1:7571518563459451198:2114]) is too old
2025-11-11T17:19:01.060774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518563459451189:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.060809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518563459451189:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.071123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518563459451175:2098] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.071176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:01.071222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518563459451175:2098] [node 4] ICP22 created new session: [1:7571518567754418503:2098]
2025-11-11T17:19:01.074503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518559164483826:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(331839315ns) {Configure:104(331832360ns) {ProcessConfigured:135(8127035ns) {StartInitialHandshake:157(8125412ns) {DropHandshakes:469(2402ns) {DropIncomingHandshake:441(347ns) DropOutgoingHandshake:456(554ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:476(2310ns) {SwitchToState:178(1010ns)}}}}}
2025-11-11T17:19:01.077702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518567754418503:2098] [node 4] ICS09 handshake done sender: [1:7571518563459451177:2099] self: [1:7571518563459451176:0] peer: [4:7571518564570127677:0] socket: 206 qp: -1
2025-11-11T17:19:01.077760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518567754418503:2098] [node 4] ICS10 traffic start
2025-11-11T17:19:01.103637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:01.122095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518563459451175:2098] [node 4] ICP35 event processing took too much time PendingConnection:278(50779143ns) {HandleHandshakeStatus:329(50775365ns) {DropIncomingHandshake:441(928ns) DropOutgoingHandshake:456(925ns) SwitchToState:178(618ns) ProcessPendingSessionEvents:495(119941ns) {ForwardSessionEventToSession:612(70489ns) ForwardSessionEventToSession:612(3133ns) ForwardSessionEventToSession:612(2804ns) ForwardSessionEventToSession:612(5905ns) ForwardSessionEventToSession:612(2641ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(8664ns) ForwardSessionEventToSession:612(2091ns) ForwardSessionEventToSession:612(6581ns) ForwardSessionEventToSession:612(2190ns)}}}
2025-11-11T17:19:01.122739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518563459451202:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.134010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518567754418505:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1312
2025-11-11T17:19:01.134221Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518563459451189:2109] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:01.134499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518563459451175:2098] [node 4] ICP18 handshake (actor [1:7571518563459451202:2115]) is too old
2025-11-11T17:19:01.134618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518567754418507:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1312
2025-11-11T17:19:01.136667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518563459451186:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.136701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:01.136743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518563459451186:2100] [node 5] ICP22 created new session: [1:7571518567754418509:2100]
2025-11-11T17:19:01.136783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518567754418509:2100] [node 5] ICS09 handshake done sender: [1:7571518563459451189:2109] self: [1:7571518563459451188:0] peer: [5:7571518561549419474:0] socket: 204 qp: -1
2025-11-11T17:19:01.136824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518567754418509:2100] [node 5] ICS10 traffic start
2025-11-11T17:19:01.136853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:01.063605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.074322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518568335503190:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26107
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.168176Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518567754418505:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.094008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518565879076938:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.126785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518565844386775:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:01.097863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518568335503190:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.097959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518564040535888:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.097979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518564040535888:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.100194Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518564040535888:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:01.100516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518564040535873:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.100575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:01.100621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518564040535873:2074] [node 6] ICP22 created new session: [3:7571518568335503194:2074]
2025-11-11T17:19:01.100660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518568335503194:2074] [node 6] ICS09 handshake done sender: [3:7571518564040535888:2114] self: [3:7571518564040535887:0] peer: [6:7571518565879076942:0] socket: 232 qp: -1
2025-11-11T17:19:01.100692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518568335503194:2074] [node 6] ICS10 traffic start
2025-11-11T17:19:01.100726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:01.114246Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518564040535836:2069] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:01.135591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518568335503197:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26107
2025-11-11T17:19:01.158418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518564040535834:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.158474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:01.158516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518564040535834:2068] [node 5] ICP22 created new session: [3:7571518568335503200:2068]
2025-11-11T17:19:01.158557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518568335503200:2068] [node 5] ICS09 handshake done sender: [3:7571518564040535836:2069] self: [3:7571518564040535835:0] peer: [5:7571518565844386771:0] socket: 199 qp: -1
2025-11-11T17:19:01.158587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518568335503200:2068] [node 5] ICS10 traffic start
2025-11-11T17:19:01.158613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:01.159060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518564040535884:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:01.064781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518562595483505:2066] [node 5] ICP22 created new session: [2:7571518566890450875:2066]
2025-11-11T17:19:00.929298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518564570127601:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:00.960406Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:00.985906Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518564570127656:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.081524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518564570127602:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(339827680ns) {Configure:104(339822487ns) {ProcessConfigured:135(20062061ns) {StartInitialHandshake:157(20059142ns) {DropHandshakes:469(2964ns) {DropIncomingHandshake:441(568ns) DropOutgoingHandshake:456(831ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:476(3554ns) {SwitchToState:178(1193ns)}}}}}
2025-11-11T17:19:01.102319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518564570127681:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:01.125688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518564570127601:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518564570127676:2114] held: no
2025-11-11T17:19:01.135198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518568865094980:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:01.135989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518568865094981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:01.136364Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518564570127655:2096] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:01.150067Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:01.165773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518568865094983:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:01.165876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518564570127600:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.165940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:01.165982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518564570127600:2048] [node 1] ICP22 created new session: [4:7571518568865094985:2048]
2025-11-11T17:19:01.166031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518568865094985:2048] [node 1] ICS09 handshake done sender: [4:7571518564570127655:2096] self: [4:7571518564570127677:0] peer: [1:7571518563459451176:0] socket: 213 qp: -1
2025-11-11T17:19:01.166066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518568865094985:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:01.166101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:01.166375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518564570127600:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:01.094340Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518565879076937:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.177727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518568335503198:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26107
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.185710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518564570127601:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(52995869ns) {HandleHandshakeStatus:395(52991190ns) {LogHandshakeFail:472(16317ns) {UpdateErrorStateLog:544(5144ns)} DropOutgoingHandshake:456(2438ns) ProcessPendingSessionEvents:495(15962231ns) {DropSessionEvent:512(11816ns) DropSessionEvent:512(4747ns) DropSessionEvent:512(4731ns) DropSessionEvent:512(5306ns) DropSessionEvent:512(4469ns) DropSessionEvent:512(7911685ns) DropSessionEvent:512(7996948ns) DropSessionEvent:512(4630ns) DropSessionEvent:512(4687ns) DropSessionEvent:512(3794ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.126992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518561549419393:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.074496Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518562595483557:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.094594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518561584109565:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.184254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518564040535814:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.203365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518564570127601:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17567548ns)
2025-11-11T17:19:01.206369Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518564570127676:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:01.209880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518564570127601:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.209918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:01.209963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518564570127601:2049] [node 2] ICP22 created new session: [4:7571518568865094988:2049]
2025-11-11T17:19:01.210027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518568865094988:2049] [node 2] ICS09 handshake done sender: [4:7571518564570127676:2114] self: [4:7571518564570127679:0] peer: [2:7571518562595483562:0] socket: 223 qp: -1
2025-11-11T17:19:01.210060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518568865094988:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:01.210093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:01.206712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518567754418512:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1312
2025-11-11T17:19:01.213759Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:01.102732Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64357) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:01.137746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518561584109564:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(315219393ns) {Configure:104(315208788ns) {ProcessConfigured:135(24877084ns) {StartInitialHandshake:157(24873402ns) {DropHandshakes:469(1416ns) {DropIncomingHandshake:441(307ns) DropOutgoingHandshake:456(208ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:476(4844ns) {SwitchToState:178(1868ns)}}}}}
2025-11-11T17:19:01.145747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518561584109565:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64357) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518565879076938:2118] held: no
2025-11-11T17:19:01.182842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518561584109631:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.184660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518561584109564:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.203777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518561584109618:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.217640Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518565879076936:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.074533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518562595483557:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.128032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518561549419406:2061] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:01.185768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:01.224290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076943:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:01.224332Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518568865094983:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.224458Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518568865094981:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.224508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518564570127602:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.225350Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518564570127615:2060] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.225374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518564570127615:2060] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.227796Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518564570127674:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.227824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518564570127674:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.231048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518568865094992:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3066
2025-11-11T17:19:01.232576Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518568865094983:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:01.232731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518564570127602:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.232770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:01.232808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518564570127602:2050] [node 3] ICP22 created new session: [4:7571518568865094993:2050]
2025-11-11T17:19:01.232843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518568865094993:2050] [node 3] ICS09 handshake done sender: [4:7571518568865094983:2121] self: [4:7571518568865094990:0] peer: [3:7571518564040535885:0] socket: 252 qp: -1
2025-11-11T17:19:01.232872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518568865094993:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:01.232903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:01.250245Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518564570127674:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.097799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518562595483557:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.226001Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518567754418502:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.224931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518561584109626:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.129800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:01.202472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518568335503199:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26107
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.263630Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518564570127615:2060] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:01.103601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518566890450875:2066] [node 5] ICS09 handshake done sender: [2:7571518562595483507:2067] self: [2:7571518562595483506:0] peer: [5:7571518561549419471:0] socket: 209 qp: -1
2025-11-11T17:19:01.127193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518562595483486:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.127224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:01.127272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518562595483486:2049] [node 3] ICP22 created new session: [2:7571518566890450877:2049]
2025-11-11T17:19:01.127307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518566890450877:2049] [node 3] ICS09 handshake done sender: [2:7571518562595483557:2112] self: [2:7571518562595483556:0] peer: [3:7571518568335503189:0] socket: 211 qp: -1
2025-11-11T17:19:01.127338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518566890450877:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:01.127370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:01.133637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518566890450875:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:01.133731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:01.176201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518562595483505:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(112692227ns) {HandleHandshakeStatus:329(112687987ns) {DropIncomingHandshake:441(24151ns) DropOutgoingHandshake:456(695ns) SwitchToState:178(540ns) ProcessPendingSessionEvents:495(6254853ns) {ForwardSessionEventToSession:612(6200799ns) ForwardSessionEventToSession:612(7932ns) ForwardSessionEventToSession:612(3212ns) ForwardSessionEventToSession:612(4241ns) ForwardSessionEventToSession:612(4775ns) ForwardSessionEventToSession:612(5246ns) ForwardSessionEventToSession:612(7894ns) ForwardSessionEventToSession:612(3989ns) ForwardSessionEventToSession:612(1735ns) ForwardSessionEventToSession:612(5125ns)}}}
2025-11-11T17:19:01.240824Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518566890450871:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.240849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518566890450871:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.270720Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518566890450871:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:01.284531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518562595483552:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.284558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:01.284593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518562595483552:2069] [node 6] ICP22 created new session: [2:7571518566890450881:2069]
2025-11-11T17:19:01.284625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518566890450881:2069] [node 6] ICS09 handshake done sender: [2:7571518566890450871:2118] self: [2:7571518566890450870:0] peer: [6:7571518565879076945:0] socket: 230 qp: -1
2025-11-11T17:19:01.284649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518566890450881:2069] [node 6] ICS10 traffic start
2025-11-11T17:19:01.284674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:01.203032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518568335503203:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26107
2025-11-11T17:19:01.145885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518561549419393:2049] [node 2] ICP22 created new session: [5:7571518565844386776:2049]
2025-11-11T17:19:01.224975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518561584109563:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.225775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518561584109565:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(26775598ns) {HandleHandshakeStatus:395(26772212ns) {LogHandshakeFail:472(13071ns) {UpdateErrorStateLog:544(5136ns)} DropOutgoingHandshake:456(2050ns) ProcessPendingSessionEvents:495(18783735ns) {DropSessionEvent:512(10241ns) DropSessionEvent:512(4473ns) DropSessionEvent:512(21465ns) DropSessionEvent:512(6091ns) DropSessionEvent:512(13570006ns) DropSessionEvent:512(9617ns) DropSessionEvent:512(3595ns) DropSessionEvent:512(3563ns) DropSessionEvent:512(3326ns) DropSessionEvent:512(3785ns)}}}
2025-11-11T17:19:01.238511Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518565879076938:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:01.273861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076950:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:01.280845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076949:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:01.290769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518561584109565:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.290810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:01.290850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518561584109565:2050] [node 3] ICP22 created new session: [6:7571518565879076953:2050]
2025-11-11T17:19:01.290892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518565879076953:2050] [node 3] ICS09 handshake done sender: [6:7571518565879076938:2118] self: [6:7571518565879076942:0] peer: [3:7571518564040535887:0] socket: 236 qp: -1
2025-11-11T17:19:01.290921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518565879076953:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:01.290953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:01.291548Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518561584109574:2058] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:01.292129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076951:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:01.294052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518565879076952:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34878
2025-11-11T17:19:01.226035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518567754418502:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.151254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518561549419394:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.282887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518564570127624:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.282937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:01.282977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518564570127624:2062] [node 5] ICP22 created new session: [4:7571518568865094997:2062]
2025-11-11T17:19:01.283017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518568865094997:2062] [node 5] ICS09 handshake done sender: [4:7571518564570127674:2113] self: [4:7571518564570127673:0] peer: [5:7571518565844386784:0] socket: 253 qp: -1
2025-11-11T17:19:01.283044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518568865094997:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:01.283073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:01.283308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518564570127613:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.283371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:01.283402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518564570127613:2059] [node 6] ICP22 created new session: [4:7571518568865094999:2059]
2025-11-11T17:19:01.283446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518568865094999:2059] [node 6] ICS09 handshake done sender: [4:7571518564570127615:2060] self: [4:7571518564570127614:0] peer: [6:7571518565879076944:0] socket: 200 qp: -1
2025-11-11T17:19:01.283470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518568865094999:2059] [node 6] ICS10 traffic start
2025-11-11T17:19:01.283496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:01.209688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518564040535814:2049] [node 2] ICP22 created new session: [3:7571518568335503202:2049]
2025-11-11T17:19:01.226614Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518567754418507:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.313995Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:01.322380Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64011) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:01.337791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518561584109631:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.337867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:01.337913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518561584109631:2072] [node 4] ICP22 created new session: [6:7571518565879076955:2072]
2025-11-11T17:19:01.337958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518565879076955:2072] [node 4] ICS09 handshake done sender: [6:7571518561584109574:2058] self: [6:7571518565879076944:0] peer: [4:7571518564570127614:0] socket: 201 qp: -1
2025-11-11T17:19:01.337998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518565879076955:2072] [node 4] ICS10 traffic start
2025-11-11T17:19:01.338037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:01.338340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518561584109631:2072] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:01.338823Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518565879076936:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:01.339314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518561584109563:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518565879076936:2116] held: no
2025-11-11T17:19:01.339395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518561584109563:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.339427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:01.339464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518561584109563:2048] [node 1] ICP22 created new session: [6:7571518565879076957:2048]
2025-11-11T17:19:01.339515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518565879076957:2048] [node 1] ICS09 handshake done sender: [6:7571518565879076936:2116] self: [6:7571518565879076948:0] peer: [1:7571518563459451190:0] socket: 202 qp: -1
2025-11-11T17:19:01.339543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518565879076957:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:01.339573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:01.339951Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518565879076937:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:01.340514Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518561584109618:2102] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:01.157031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518565844386776:2049] [node 2] ICS09 handshake done sender: [5:7571518561549419430:2080] self: [5:7571518561549419471:0] peer: [2:7571518562595483506:0] socket: 214 qp: -1
2025-11-11T17:19:01.217621Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:01.340905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518561584109564:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.340944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:01.340973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518561584109564:2049] [node 2] ICP22 created new session: [6:7571518565879076959:2049]
2025-11-11T17:19:01.341007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518565879076959:2049] [node 2] ICS09 handshake done sender: [6:7571518565879076937:2117] self: [6:7571518565879076945:0] peer: [2:7571518566890450870:0] socket: 235 qp: -1
2025-11-11T17:19:01.341032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518565879076959:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:01.341058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:01.345934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518561584109626:2070] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.345986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:01.346027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518561584109626:2070] [node 5] ICP22 created new session: [6:7571518565879076961:2070]
2025-11-11T17:19:01.346066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518565879076961:2070] [node 5] ICS09 handshake done sender: [6:7571518561584109618:2102] self: [6:7571518565879076947:0] peer: [5:7571518561549419418:0] socket: 219 qp: -1
2025-11-11T17:19:01.346099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518565879076961:2070] [node 5] ICS10 traffic start
2025-11-11T17:19:01.346128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:01.157075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518565844386776:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:01.157131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:01.157374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518561549419393:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(30306939ns) {HandleHandshakeStatus:329(30304390ns) {DropIncomingHandshake:441(828ns) DropOutgoingHandshake:456(272ns) SwitchToState:178(629ns) ProcessPendingSessionEvents:495(653ns)}}
2025-11-11T17:19:01.169678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:01.178588Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518561549419408:2063] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:01.179528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518561549419394:2050] [node 3] ICP22 created new session: [5:7571518565844386777:2050]
2025-11-11T17:19:01.185726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518565844386777:2050] [node 3] ICS09 handshake done sender: [5:7571518561549419406:2061] self: [5:7571518565844386771:0] peer: [3:7571518564040535835:0] socket: 203 qp: -1
2025-11-11T17:19:01.193583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518561549419392:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.193650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:01.193700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518561549419392:2048] [node 1] ICP22 created new session: [5:7571518565844386781:2048]
2025-11-11T17:19:01.193747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518565844386781:2048] [node 1] ICS09 handshake done sender: [5:7571518561549419408:2063] self: [5:7571518561549419474:0] peer: [1:7571518563459451188:0] socket: 205 qp: -1
2025-11-11T17:19:01.193789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518565844386781:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:01.193826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:01.194459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518565844386779:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:01.197769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518565844386777:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:01.205897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:01.207060Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518565844386779:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.207218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518561549419413:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.240728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518561549419419:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.240762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518561549419419:2070] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.258716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518565844386785:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:01.261023Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518561549419419:2070] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:01.261308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518561549419415:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.261355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:01.261409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518561549419415:2067] [node 6] ICP22 created new session: [5:7571518565844386787:2067]
2025-11-11T17:19:01.261459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518565844386787:2067] [node 6] ICS09 handshake done sender: [5:7571518561549419419:2070] self: [5:7571518561549419418:0] peer: [6:7571518565879076947:0] socket: 216 qp: -1
2025-11-11T17:19:01.261498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518565844386787:2067] [node 6] ICS10 traffic start
2025-11-11T17:19:01.261538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:01.284095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518565844386789:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1629
2025-11-11T17:19:01.297915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518561549419415:2067] [node 6] ICP35 event processing took too much time PendingConnection:278(36412223ns) {HandleHandshakeStatus:329(36408847ns) {DropIncomingHandshake:441(714ns) DropOutgoingHandshake:456(792ns) SwitchToState:178(938ns) ProcessPendingSessionEvents:495(36022571ns) {ForwardSessionEventToSession:612(12562472ns) ForwardSessionEventToSession:612(7398249ns) ForwardSessionEventToSession:612(2674ns) ForwardSessionEventToSession:612(8111944ns) ForwardSessionEventToSession:612(7311ns) ForwardSessionEventToSession:612(2956ns) ForwardSessionEventToSession:612(7896026ns) ForwardSessionEventToSession:612(6702ns) ForwardSessionEventToSession:612(2825ns) ForwardSessionEventToSession:612(8875ns)}}}
2025-11-11T17:19:01.298046Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64011) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:01.321744Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518565844386779:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:01.341667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518561549419394:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(143183446ns) {HandleHandshakeStatus:329(143179742ns) {DropIncomingHandshake:441(937ns) DropOutgoingHandshake:456(5602ns) SwitchToState:178(615ns) ProcessPendingSessionEvents:495(52733962ns) {ForwardSessionEventToSession:612(20031190ns) ForwardSessionEventToSession:612(3994ns) ForwardSessionEventToSession:612(2402ns) ForwardSessionEventToSession:612(8069438ns) ForwardSessionEventToSession:612(3616ns) ForwardSessionEventToSession:612(7838864ns) ForwardSessionEventToSession:612(8510086ns) ForwardSessionEventToSession:612(2870ns) ForwardSessionEventToSession:612(8234979ns) ForwardSessionEventToSession:612(7646ns)}}}
2025-11-11T17:19:01.217712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518568335503199:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.217853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518568335503202:2049] [node 2] ICS09 handshake done sender: [3:7571518564040535884:2112] self: [3:7571518568335503189:0] peer: [2:7571518562595483556:0] socket: 227 qp: -1
2025-11-11T17:19:01.217890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518568335503202:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:01.217949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518564040535834:2068] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:01.218058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518564040535813:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:01.224168Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518568335503203:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.225736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:01.229733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518564040535886:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.229752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518564040535886:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.230736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518568335503208:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26107
2025-11-11T17:19:01.231221Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518564040535886:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:01.231459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518564040535878:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.231510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:01.231553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518564040535878:2066] [node 4] ICP22 created new session: [3:7571518568335503210:2066]
2025-11-11T17:19:01.231602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518568335503210:2066] [node 4] ICS09 handshake done sender: [3:7571518564040535886:2113] self: [3:7571518564040535885:0] peer: [4:7571518568865094990:0] socket: 250 qp: -1
2025-11-11T17:19:01.231633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518568335503210:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:01.231666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:01.242101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518564040535814:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(57627101ns) {HandleHandshakeStatus:329(57624255ns) {DropIncomingHandshake:441(701ns) DropOutgoingHandshake:456(4599ns) SwitchToState:178(261ns) ProcessPendingSessionEvents:495(96797ns) {ForwardSessionEventToSession:612(48794ns) ForwardSessionEventToSession:612(2926ns) ForwardSessionEventToSession:612(2815ns) ForwardSessionEventToSession:612(5881ns) ForwardSessionEventToSession:612(3042ns) ForwardSessionEventToSession:612(8159ns) ForwardSessionEventToSession:612(4452ns) ForwardSessionEventToSession:612(2529ns) ForwardSessionEventToSession:612(8410ns) ForwardSessionEventToSession:612(2490ns)}}}
2025-11-11T17:19:01.265987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518568335503199:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:01.299072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518564040535813:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.299116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:01.299177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518564040535813:2048] [node 1] ICP22 created new session: [3:7571518568335503212:2048]
2025-11-11T17:19:01.299223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518568335503212:2048] [node 1] ICS09 handshake done sender: [3:7571518568335503199:2119] self: [3:7571518568335503205:0] peer: [1:7571518567754418501:0] socket: 226 qp: -1
2025-11-11T17:19:01.299255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518568335503212:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:01.299303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:01.309806Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:01.325793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518564040535813:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:01.226734Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518563459451191:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.226752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518563459451191:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:01.355112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518561549419413:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64011) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518565844386779:2121] held: no
2025-11-11T17:19:01.356356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518561549419413:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:01.356398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:01.358324Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518565844386785:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:01.362265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518561549419413:2060] [node 4] ICP22 created new session: [5:7571518565844386790:2060]
2025-11-11T17:19:01.363154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518561549419415:2067] [node 6] ICP18 handshake (actor [5:7571518565844386785:2122]) is too old
2025-11-11T17:19:01.365865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518565844386790:2060] [node 4] ICS09 handshake done sender: [5:7571518565844386779:2121] self: [5:7571518565844386784:0] peer: [4:7571518564570127673:0] socket: 217 qp: -1
2025-11-11T17:19:01.365904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518565844386790:2060] [node 4] ICS10 traffic start
2025-11-11T17:19:01.365947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:01.234017Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518563459451191:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:01.237873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518567754418502:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:01.266120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518563459451187:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.266182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:01.266227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518563459451187:2101] [node 6] ICP22 created new session: [1:7571518567754418516:2101]
2025-11-11T17:19:01.266272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518567754418516:2101] [node 6] ICS09 handshake done sender: [1:7571518563459451191:2110] self: [1:7571518563459451190:0] peer: [6:7571518565879076948:0] socket: 228 qp: -1
2025-11-11T17:19:01.266300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518567754418516:2101] [node 6] ICS10 traffic start
2025-11-11T17:19:01.266330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:01.281966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518559164483826:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:01.282026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:01.282063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518559164483826:2049] [node 3] ICP22 created new session: [1:7571518567754418518:2049]
2025-11-11T17:19:01.282114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518567754418518:2049] [node 3] ICS09 handshake done sender: [1:7571518567754418502:2116] self: [1:7571518567754418501:0] peer: [3:7571518568335503205:0] socket: 254 qp: -1
2025-11-11T17:19:01.282147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518567754418518:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:01.282176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:01.401760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:01.402014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518562595483567:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-11-11T17:19:01.402048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518562595483485:2048] [node 1] ICP30 unregister session Session# [2:7571518562595483567:2048] VirtualId# [2:7571518562595483565:0]
2025-11-11T17:19:01.402078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518562595483567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.402155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:01.402190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:01.403765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:01.403957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518568335503212:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-11-11T17:19:01.403987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518564040535813:2048] [node 1] ICP30 unregister session Session# [3:7571518568335503212:2048] VirtualId# [3:7571518568335503205:0]
2025-11-11T17:19:01.404009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518568335503212:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.404061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:01.404094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:01.409755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:01.409996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518565844386781:2048] [node 1] ICS01 socket: 205 reason# EndOfStream
2025-11-11T17:19:01.410030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518561549419392:2048] [node 1] ICP30 unregister session Session# [5:7571518565844386781:2048] VirtualId# [5:7571518561549419474:0]
2025-11-11T17:19:01.410058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518565844386781:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.410122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:01.410158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:01.409799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:01.410025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518568865094985:2048] [node 1] ICS01 socket: 213 reason# EndOfStream
2025-11-11T17:19:01.410060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518564570127600:2048] [node 1] ICP30 unregister session Session# [4:7571518568865094985:2048] VirtualId# [4:7571518564570127677:0]
2025-11-11T17:19:01.410085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518568865094985:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.410155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:01.410191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:01.410229Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:01.410414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518565879076957:2048] [node 1] ICS01 socket: 202 reason# EndOfStream
2025-11-11T17:19:01.410444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518561584109563:2048] [node 1] ICP30 unregister session Session# [6:7571518565879076957:2048] VirtualId# [6:7571518565879076948:0]
2025-11-11T17:19:01.410465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518565879076957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.410501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:01.410529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:01.479307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:01.479546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518568335503202:2049] [node 2] ICS01 socket: 227 reason# EndOfStream
2025-11-11T17:19:01.479584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518564040535814:2049] [node 2] ICP30 unregister session Session# [3:7571518568335503202:2049] VirtualId# [3:7571518568335503189:0]
2025-11-11T17:19:01.479616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518568335503202:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.479680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:01.479718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:01.522811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:01.522996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518565879076959:2049] [node 2] ICS07 socket disconnect 235 reason# EndOfStream
2025-11-11T17:19:01.523020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518565879076959:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.523062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:01.523096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518565879076959:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:01.521865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:01.522122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518568865094988:2049] [node 2] ICS01 socket: 223 reason# EndOfStream
2025-11-11T17:19:01.522158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518564570127601:2049] [node 2] ICP30 unregister session Session# [4:7571518568865094988:2049] VirtualId# [4:7571518564570127679:0]
2025-11-11T17:19:01.522189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518568865094988:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.522290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:01.522326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:01.522403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:01.522609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518565844386776:2049] [node 2] ICS01 socket: 214 reason# EndOfStream
2025-11-11T17:19:01.522639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518561549419393:2049] [node 2] ICP30 unregister session Session# [5:7571518565844386776:2049] VirtualId# [5:7571518561549419471:0]
2025-11-11T17:19:01.522660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518565844386776:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.522704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:01.522737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:01.557772Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Couldn't connect to any resolved address
2025-11-11T17:19:01.558090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518561584109564:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:01.558141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518565879076959:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:01.558173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518561584109564:2049] [node 2] ICP30 unregister session Session# [6:7571518565879076959:2049] VirtualId# [6:7571518565879076945:0]
2025-11-11T17:19:01.558217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:01.558247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518561584109564:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:01.558261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:01.573674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518561584109564:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:01.677805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:01.678009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518568865094993:2050] [node 3] ICS01 socket: 252 reason# EndOfStream
2025-11-11T17:19:01.678050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518564570127602:2050] [node 3] ICP30 unregister session Session# [4:7571518568865094993:2050] VirtualId# [4:7571518568865094990:0]
2025-11-11T17:19:01.678081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518568865094993:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.678174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:01.678229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:01.677778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:01.678014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518565844386777:2050] [node 3] ICS07 socket disconnect 203 reason# EndOfStream
2025-11-11T17:19:01.678040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518565844386777:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.678831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:01.701709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518565879076953:2050] [node 3] ICS01 socket: 236 reason# EndOfStream
2025-11-11T17:19:01.713726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518561584109565:2050] [node 3] ICP30 unregister session Session# [6:7571518565879076953:2050] VirtualId# [6:7571518565879076942:0]
2025-11-11T17:19:01.713791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518565879076953:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.713930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:01.713977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:01.678147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:01.678187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518565844386777:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:01.685780Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64357) Couldn't connect to any resolved address
2025-11-11T17:19:01.686132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518561549419394:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:01.686181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518565844386777:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:01.686217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518561549419394:2050] [node 3] ICP30 unregister session Session# [5:7571518565844386777:2050] VirtualId# [5:7571518565844386771:0]
2025-11-11T17:19:01.686265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:01.686293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518561549419394:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:01.686308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:01.701685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518561549419394:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:01.745743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:01.745744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:01.745991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518565844386790:2060] [node 4] ICS01 socket: 217 reason# EndOfStream
2025-11-11T17:19:01.746032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518561549419413:2060] [node 4] ICP30 unregister session Session# [5:7571518565844386790:2060] VirtualId# [5:7571518565844386784:0]
2025-11-11T17:19:01.746064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518565844386790:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.746151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:01.746190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:01.746048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518565879076955:2072] [node 4] ICS07 socket disconnect 201 reason# EndOfStream
2025-11-11T17:19:01.746074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518565879076955:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.746145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:01.746186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518565879076955:2072] [node 4] ICS15 start handshake
2025-11-11T17:19:01.747031Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64011) Couldn't connect to any resolved address
2025-11-11T17:19:01.747269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518561584109631:2072] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64011) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:01.747315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518565879076955:2072] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:01.747457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518561584109631:2072] [node 4] ICP30 unregister session Session# [6:7571518565879076955:2072] VirtualId# [6:7571518565879076944:0]
2025-11-11T17:19:01.747500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:01.747533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518561584109631:2072] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64011) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:01.747546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64011) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:01.761692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518561584109631:2072] [node 4] ICP33 wake up from error state
2025-11-11T17:19:01.846959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:01.847222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518565879076961:2070] [node 5] ICS01 socket: 219 reason# EndOfStream
2025-11-11T17:19:01.847263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518561584109626:2070] [node 5] ICP30 unregister session Session# [6:7571518565879076961:2070] VirtualId# [6:7571518565879076947:0]
2025-11-11T17:19:01.847300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518565879076961:2070] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:01.847414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:01.847458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:01.981913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518566764971571:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:01.981958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518566764971572:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:02.041880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518573323305335:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:02.041923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518573323305336:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:02.101954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518569926216179:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:02.102001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518569926216180:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:02.149934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518570402942227:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:02.150000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518570402942228:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:02.150033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518570402942229:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:02.187955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518572460652992:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:02.187994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518572460652993:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:02.188018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518572460652994:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.326456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518570402942241:2060] [node 5] ICP01 ready to work
2025-11-11T17:19:02.358686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518571228282885:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:02.358721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518571228282886:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:02.358784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518571228282887:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:02.382137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518571228282902:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:02.384437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518570402942246:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65352
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.562683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518571228282910:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:02.497941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518573323305368:2063] [node 5] ICP01 ready to work
2025-11-11T17:19:02.501676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518571059938912:2067] [node 5] ICP01 ready to work
2025-11-11T17:19:02.529980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518572460653013:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.554126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518573323305399:2088] [node 6] ICP01 ready to work
2025-11-11T17:19:02.637641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518569926216241:2058] [node 5] ICP01 ready to work
2025-11-11T17:19:02.501674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518571059938913:2066] [node 6] ICP01 ready to work
2025-11-11T17:19:02.534021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518572460653012:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:02.706017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518569926216243:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:02.695916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282955:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:02.749714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518570402942298:2094] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.557519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518573323305398:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26678
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.521768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518571059938917:2065] [node 4] ICP01 ready to work
2025-11-11T17:19:02.637646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518572460653019:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:02.577755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518573323305394:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:02.746453Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518571228282955:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.757426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518569926216245:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.814679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518566764971572:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(472701502ns) {Configure:104(444258051ns) {ProcessConfigured:135(56999ns) {StartInitialHandshake:157(54868ns) {DropHandshakes:469(3288ns) {DropIncomingHandshake:441(699ns) DropOutgoingHandshake:456(730ns)} GenerateSessionVirtualId:988(1083ns) PrepareNewSessionHandshake:476(1737ns) {SwitchToState:178(612ns)}}}}}
2025-11-11T17:19:02.816401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518571059938935:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32040
2025-11-11T17:19:02.749933Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518573323305403:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.781929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518570402942246:2063] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.664679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518572460653013:2067] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.746586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518571228282886:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:02.757556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518569926216246:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37729
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.785680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518570402942299:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65352
2025-11-11T17:19:02.754187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282962:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:02.749956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518573323305403:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.702707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518572460653056:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
2025-11-11T17:19:02.766660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518569926216244:2073] [node 4] ICP01 ready to work
2025-11-11T17:19:02.821909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518571059938935:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.754290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282963:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:02.750732Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518573323305403:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:02.806207Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518570402942299:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.828280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518570402942302:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65352
2025-11-11T17:19:02.833875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518570402942227:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(475385799ns) {Configure:104(475381407ns) {ProcessConfigured:135(40320ns) {StartInitialHandshake:157(37983ns) {DropHandshakes:469(1529ns) {DropIncomingHandshake:441(450ns) DropOutgoingHandshake:456(207ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:476(653ns) {SwitchToState:178(281ns)}}}}}
2025-11-11T17:19:02.845763Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518570402942302:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.853852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518570402942227:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:02.702760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518572460653057:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.836898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518571059938940:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32040
2025-11-11T17:19:02.789626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518569926216247:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.893684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518571059938937:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.893721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518571059938937:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:02.894722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518571059938942:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32040
2025-11-11T17:19:02.897261Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518571059938937:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:02.901960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518571059938917:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:02.757727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518571228282962:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.757878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518571228282910:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:02.758059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518571228282885:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(193041025ns) {Configure:104(193038033ns) {ProcessConfigured:135(16089005ns) {StartInitialHandshake:157(16086869ns) {DropHandshakes:469(3108ns) {DropIncomingHandshake:441(746ns) DropOutgoingHandshake:456(614ns)} GenerateSessionVirtualId:988(1222ns) PrepareNewSessionHandshake:476(967ns) {SwitchToState:178(121ns)}}}}}
2025-11-11T17:19:02.821874Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:02.826270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518571228282955:2108] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:02.826423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518571228282886:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518571228282955:2108] held: no
2025-11-11T17:19:02.826719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:02.837303Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:02.855181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518571228282886:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:02.855247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:02.855293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518571228282886:2049] [node 2] ICP22 created new session: [6:7571518571228282969:2049]
2025-11-11T17:19:02.861844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518571228282910:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518571228282962:2114] held: no
2025-11-11T17:19:02.861888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518571228282969:2049] [node 2] ICS09 handshake done sender: [6:7571518571228282955:2108] self: [6:7571518571228282961:0] peer: [2:7571518573323305402:0] socket: 216 qp: -1
2025-11-11T17:19:02.861941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518571228282969:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:02.861996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:02.870076Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518571228282962:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:02.870688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518571228282910:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:02.870737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:02.870787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518571228282910:2066] [node 5] ICP22 created new session: [6:7571518571228282973:2066]
2025-11-11T17:19:02.870833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518571228282973:2066] [node 5] ICS09 handshake done sender: [6:7571518571228282962:2114] self: [6:7571518571228282964:0] peer: [5:7571518572460653061:0] socket: 222 qp: -1
2025-11-11T17:19:02.870867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518571228282973:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:02.870903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:02.902427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282975:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.895236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518570402942307:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65352
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.750979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518573323305399:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:02.720146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518572460653056:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.770021Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518572460653057:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.770141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518572460653062:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.770163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518572460653062:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:02.771044Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518572460653062:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:02.771248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518572460653019:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:02.771300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:02.771345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518572460653019:2065] [node 6] ICP22 created new session: [5:7571518572460653071:2065]
2025-11-11T17:19:02.771398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518572460653071:2065] [node 6] ICS09 handshake done sender: [5:7571518572460653062:2112] self: [5:7571518572460653061:0] peer: [6:7571518571228282964:0] socket: 219 qp: -1
2025-11-11T17:19:02.771426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518572460653071:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:02.771460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:02.846092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518572460653077:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
2025-11-11T17:19:02.888349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518572460652993:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(415871332ns) {Configure:104(415862679ns) {ProcessConfigured:135(3564410ns) {StartInitialHandshake:157(3562493ns) {DropHandshakes:469(1519ns) {DropIncomingHandshake:441(327ns) DropOutgoingHandshake:456(428ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:476(1937ns) {SwitchToState:178(816ns)}}}}}
2025-11-11T17:19:02.889620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518572460653012:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(153184337ns) {Configure:104(153178764ns) {ProcessConfigured:135(28248507ns) {StartInitialHandshake:157(28245020ns) {DropHandshakes:469(2989ns) {DropIncomingHandshake:441(591ns) DropOutgoingHandshake:456(575ns)} GenerateSessionVirtualId:988(4433ns) PrepareNewSessionHandshake:476(2042ns) {SwitchToState:178(577ns)}}}}}
2025-11-11T17:19:02.913661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518572460653012:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:02.913957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518572460652993:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:02.821635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216246:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.751015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:02.751057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518573323305399:2088] [node 6] ICP22 created new session: [2:7571518573323305414:2088]
2025-11-11T17:19:02.751099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518573323305414:2088] [node 6] ICS09 handshake done sender: [2:7571518573323305403:2110] self: [2:7571518573323305402:0] peer: [6:7571518571228282961:0] socket: 214 qp: -1
2025-11-11T17:19:02.751132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518573323305414:2088] [node 6] ICS10 traffic start
2025-11-11T17:19:02.751163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:02.766501Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518573323305398:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.777956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518573323305399:2088] [node 6] ICP18 handshake (actor [2:7571518573323305398:2108]) is too old
2025-11-11T17:19:02.785959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518573323305336:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(432834250ns) {Configure:104(432827870ns) {ProcessConfigured:135(35901471ns) {StartInitialHandshake:157(35898909ns) {DropHandshakes:469(1555ns) {DropIncomingHandshake:441(464ns) DropOutgoingHandshake:456(180ns)} GenerateSessionVirtualId:988(605ns) PrepareNewSessionHandshake:476(4946ns) {SwitchToState:178(2058ns)}}}}}
2025-11-11T17:19:02.853744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518573323305335:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(471748906ns) {Configure:104(448325479ns) {ProcessConfigured:135(46169ns) {StartInitialHandshake:157(43694ns) {DropHandshakes:469(2107ns) {DropIncomingHandshake:441(652ns) DropOutgoingHandshake:456(423ns)} GenerateSessionVirtualId:988(325ns) PrepareNewSessionHandshake:476(1245ns) {SwitchToState:178(784ns)}}}}}
2025-11-11T17:19:02.910596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518573323305419:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26678
2025-11-11T17:19:02.915012Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518573323305401:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.915030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518573323305401:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:02.937825Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518573323305401:2109] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.913707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:02.938148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518573323305368:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:02.938201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:02.938248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518573323305368:2063] [node 5] ICP22 created new session: [2:7571518573323305421:2063]
2025-11-11T17:19:02.938289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518573323305421:2063] [node 5] ICS09 handshake done sender: [2:7571518573323305401:2109] self: [2:7571518573323305400:0] peer: [5:7571518572460653080:0] socket: 212 qp: -1
2025-11-11T17:19:02.938324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518573323305421:2063] [node 5] ICS10 traffic start
2025-11-11T17:19:02.938353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:02.939851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518573323305410:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.939873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518573323305410:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:02.940663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518573323305410:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:02.940840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518573323305394:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:02.940870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:02.940908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518573323305394:2064] [node 4] ICP22 created new session: [2:7571518573323305425:2064]
2025-11-11T17:19:02.940943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518573323305425:2064] [node 4] ICS09 handshake done sender: [2:7571518573323305410:2113] self: [2:7571518573323305409:0] peer: [4:7571518570402942315:0] socket: 220 qp: -1
2025-11-11T17:19:02.940973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518573323305425:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:02.940999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.825652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216245:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.922010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282976:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:02.895292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518570402942308:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65352
2025-11-11T17:19:02.902208Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518570402942302:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:02.902645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518570402942227:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:02.902692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:02.902734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518570402942227:2048] [node 1] ICP22 created new session: [4:7571518570402942309:2048]
2025-11-11T17:19:02.902775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518570402942309:2048] [node 1] ICS09 handshake done sender: [4:7571518570402942302:2115] self: [4:7571518570402942304:0] peer: [1:7571518571059938936:0] socket: 232 qp: -1
2025-11-11T17:19:02.902806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518570402942309:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:02.902837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:02.914216Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518570402942243:2061] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.914235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518570402942243:2061] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:02.919566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518570402942228:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(433309219ns) {Configure:104(433302942ns) {ProcessConfigured:135(9616684ns) {StartInitialHandshake:157(5736547ns) {DropHandshakes:469(2881ns) {DropIncomingHandshake:441(517ns) DropOutgoingHandshake:456(594ns)} GenerateSessionVirtualId:988(663ns) PrepareNewSessionHandshake:476(1662ns) {SwitchToState:178(132ns)}}}}}
2025-11-11T17:19:02.922423Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518570402942243:2061] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:02.939261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518570402942241:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:02.939301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:02.939338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518570402942241:2060] [node 5] ICP22 created new session: [4:7571518570402942316:2060]
2025-11-11T17:19:02.949747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518570402942228:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:02.950545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518570402942316:2060] [node 5] ICS09 handshake done sender: [4:7571518570402942243:2061] self: [4:7571518570402942242:0] peer: [5:7571518572460653079:0] socket: 200 qp: -1
2025-11-11T17:19:02.953860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518570402942228:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24021838ns) {IncomingHandshake:261(24019904ns) {DropIncomingHandshake:441(326ns) PrepareNewSessionHandshake:476(1154ns) {SwitchToState:178(361ns)} GenerateSessionVirtualId:988(295ns) IssueIncomingHandshakeReply:193(12088693ns) {SwitchToState:178(2014ns)}}}
2025-11-11T17:19:02.954196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518570402942317:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65352
2025-11-11T17:19:02.954703Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518570402942299:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:02.954859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518570402942228:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:02.954904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:02.954944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518570402942228:2049] [node 2] ICP22 created new session: [4:7571518570402942318:2049]
2025-11-11T17:19:02.955004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518570402942318:2049] [node 2] ICS09 handshake done sender: [4:7571518570402942299:2113] self: [4:7571518570402942315:0] peer: [2:7571518573323305409:0] socket: 221 qp: -1
2025-11-11T17:19:02.955041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518570402942318:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:02.955071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:02.958691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518570402942298:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(71967307ns) {Configure:104(71960807ns) {ProcessConfigured:135(14940779ns) {StartInitialHandshake:157(14936720ns) {DropHandshakes:469(3176ns) {DropIncomingHandshake:441(677ns) DropOutgoingHandshake:456(540ns)} GenerateSessionVirtualId:988(1023ns) PrepareNewSessionHandshake:476(3542ns) {SwitchToState:178(1288ns)}}}}}
2025-11-11T17:19:02.913769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518571059938917:2065] [node 4] ICP22 created new session: [1:7571518571059938944:2065]
2025-11-11T17:19:02.837686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216247:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.911625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518569926216254:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37729
2025-11-11T17:19:02.925746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518569926216255:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37729
2025-11-11T17:19:02.934060Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216254:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.946225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518572460653081:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.962791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518570402942316:2060] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.922170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282977:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:02.917880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518571059938912:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(208075406ns) {Configure:104(208070002ns) {ProcessConfigured:135(52326ns) {StartInitialHandshake:157(50053ns) {DropHandshakes:469(1232ns) {DropIncomingHandshake:441(312ns) DropOutgoingHandshake:456(241ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(733ns) {SwitchToState:178(163ns)}}}}}
2025-11-11T17:19:02.918001Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518571059938942:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.918170Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:02.918549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518571059938913:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(283889505ns) {Configure:104(283882149ns) {ProcessConfigured:135(24622026ns) {StartInitialHandshake:157(24619856ns) {DropHandshakes:469(3138ns) {DropIncomingHandshake:441(611ns) DropOutgoingHandshake:456(571ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:476(1476ns) {SwitchToState:178(264ns)}}}}}
2025-11-11T17:19:02.925675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518571059938944:2065] [node 4] ICS09 handshake done sender: [1:7571518571059938937:2112] self: [1:7571518571059938936:0] peer: [4:7571518570402942304:0] socket: 230 qp: -1
2025-11-11T17:19:02.937296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518571059938944:2065] [node 4] ICS10 traffic start
2025-11-11T17:19:02.961640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:02.994368Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518571059938934:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.994391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518571059938934:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:02.996022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518571059938954:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32040
2025-11-11T17:19:02.938061Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216255:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.009799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518571059938934:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:03.010254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518571059938917:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(108081749ns) {HandleHandshakeStatus:329(108077358ns) {DropIncomingHandshake:441(607ns) DropOutgoingHandshake:456(877ns) SwitchToState:178(540ns) ProcessPendingSessionEvents:495(12539417ns) {ForwardSessionEventToSession:612(12488001ns) ForwardSessionEventToSession:612(6808ns) ForwardSessionEventToSession:612(8667ns) ForwardSessionEventToSession:612(2848ns) ForwardSessionEventToSession:612(5033ns) ForwardSessionEventToSession:612(2373ns) ForwardSessionEventToSession:612(2741ns) ForwardSessionEventToSession:612(4859ns) ForwardSessionEventToSession:612(2485ns) ForwardSessionEventToSession:612(1828ns)}}}
2025-11-11T17:19:03.010462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518571059938917:2065] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:02.962611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518572460653083:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
2025-11-11T17:19:02.973772Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518573323305419:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.985620Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518570402942306:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.957788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518571228282976:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.973697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518572460653082:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
2025-11-11T17:19:02.989889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518569926216179:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(283594979ns) {Configure:104(283590725ns) {ProcessConfigured:135(24010514ns) {StartInitialHandshake:157(12042694ns) {DropHandshakes:469(5488ns) {DropIncomingHandshake:441(1850ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(1976ns) PrepareNewSessionHandshake:476(3787ns) {SwitchToState:178(1630ns)}}}}}
2025-11-11T17:19:02.985663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518570402942306:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:03.006259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518573323305368:2063] [node 5] ICP18 handshake (actor [2:7571518573323305419:2116]) is too old
2025-11-11T17:19:02.957951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518571228282902:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:02.991422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518569926216179:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.033987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518566764971571:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(417267348ns) {Configure:104(417259495ns) {ProcessConfigured:135(57257574ns) {StartInitialHandshake:157(57255122ns) {DropHandshakes:469(1876ns) {DropIncomingHandshake:441(459ns) DropOutgoingHandshake:456(346ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:476(3597ns) {SwitchToState:178(859ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.986730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518571228282979:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.993699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:02.981748Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518572460653013:2067] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:02.997231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518569926216267:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:02.997603Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518570402942306:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:03.034047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518566764971572:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.073833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518573323305412:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:02.996289Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518572460653056:2108] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:03.053753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518571228282975:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.029579Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518569926216254:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:03.034095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:03.073873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518573323305412:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:03.035543Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518570402942307:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.018430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518572460653012:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.074914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518577618272723:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26678
2025-11-11T17:19:03.109722Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518573323305412:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:03.110071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518573323305336:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.113670Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518577618272723:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.113779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:03.113822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518573323305336:2049] [node 3] ICP22 created new session: [2:7571518577618272725:2049]
2025-11-11T17:19:03.113864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518577618272725:2049] [node 3] ICS09 handshake done sender: [2:7571518573323305412:2114] self: [2:7571518573323305408:0] peer: [3:7571518574221183566:0] socket: 225 qp: -1
2025-11-11T17:19:03.114242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518577618272725:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:03.121686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:03.053903Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518571228282976:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:03.029823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518569926216179:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.034141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518566764971572:2049] [node 3] ICP22 created new session: [1:7571518575354906253:2049]
2025-11-11T17:19:03.170436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518573323305336:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(60144857ns) {HandleHandshakeStatus:329(60143097ns) {DropIncomingHandshake:441(674ns) DropOutgoingHandshake:456(611ns) SwitchToState:178(366ns) ProcessPendingSessionEvents:495(4561352ns) {ForwardSessionEventToSession:612(4503751ns) ForwardSessionEventToSession:612(8701ns) ForwardSessionEventToSession:612(4608ns) ForwardSessionEventToSession:612(3735ns) ForwardSessionEventToSession:612(3017ns) ForwardSessionEventToSession:612(5170ns) ForwardSessionEventToSession:612(2373ns) ForwardSessionEventToSession:612(8575ns) ForwardSessionEventToSession:612(2369ns) ForwardSessionEventToSession:612(4974ns)}}}
2025-11-11T17:19:03.170524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518573323305336:2049] [node 3] ICP18 handshake (actor [2:7571518577618272723:2117]) is too old
2025-11-11T17:19:03.045806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518570402942298:2094] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:03.181914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518577618272727:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26678
2025-11-11T17:19:03.029868Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518572460653083:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.065713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518571228282977:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.034190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518575354906253:2049] [node 3] ICS09 handshake done sender: [1:7571518571059938934:2110] self: [1:7571518571059938933:0] peer: [3:7571518569926216266:0] socket: 217 qp: -1
2025-11-11T17:19:03.081684Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18687) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:03.034224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518575354906253:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:03.034261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:03.086021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518571228282902:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.037653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518572460653077:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.049763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518572460652993:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.049820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:03.049864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518572460652993:2049] [node 2] ICP22 created new session: [5:7571518576755620380:2049]
2025-11-11T17:19:03.049907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518576755620380:2049] [node 2] ICS09 handshake done sender: [5:7571518572460653056:2108] self: [5:7571518572460653080:0] peer: [2:7571518573323305400:0] socket: 215 qp: -1
2025-11-11T17:19:03.049940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518576755620380:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:03.049973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:03.050563Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:03.057912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518572460652993:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:03.058076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518572460652992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.058143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518572460652994:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.059022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:03.059073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518572460653012:2061] [node 4] ICP22 created new session: [5:7571518576755620384:2061]
2025-11-11T17:19:03.107576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518576755620384:2061] [node 4] ICS09 handshake done sender: [5:7571518572460653013:2067] self: [5:7571518572460653079:0] peer: [4:7571518570402942242:0] socket: 201 qp: -1
2025-11-11T17:19:03.110637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518576755620385:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
2025-11-11T17:19:03.113764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518576755620384:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:03.113824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:03.119559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518572460653012:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(109750239ns) {HandleHandshakeStatus:329(109746525ns) {DropIncomingHandshake:441(1657ns) DropOutgoingHandshake:456(7321ns) SwitchToState:178(297ns) ProcessPendingSessionEvents:495(5444268ns) {ForwardSessionEventToSession:612(5397258ns) ForwardSessionEventToSession:612(8673ns) ForwardSessionEventToSession:612(2933ns) ForwardSessionEventToSession:612(4801ns) ForwardSessionEventToSession:612(4430ns) ForwardSessionEventToSession:612(2169ns) ForwardSessionEventToSession:612(5692ns) ForwardSessionEventToSession:612(1982ns) ForwardSessionEventToSession:612(2324ns) ForwardSessionEventToSession:612(4250ns)}}}
2025-11-11T17:19:03.125815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518576755620386:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8522
2025-11-11T17:19:03.126620Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518572460653083:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:03.128220Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518572460653077:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:03.128366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518572460652992:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.128401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:03.128442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518572460652992:2048] [node 1] ICP22 created new session: [5:7571518576755620388:2048]
2025-11-11T17:19:03.128483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518576755620388:2048] [node 1] ICS09 handshake done sender: [5:7571518572460653083:2121] self: [5:7571518576755620382:0] peer: [1:7571518571059938945:0] socket: 258 qp: -1
2025-11-11T17:19:03.128510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518576755620388:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:03.128537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:03.128778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518572460652994:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.128803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:03.128828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518572460652994:2050] [node 3] ICP22 created new session: [5:7571518576755620390:2050]
2025-11-11T17:19:03.128855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518576755620390:2050] [node 3] ICS09 handshake done sender: [5:7571518572460653077:2118] self: [5:7571518576755620383:0] peer: [3:7571518569926216248:0] socket: 237 qp: -1
2025-11-11T17:19:03.128879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518576755620390:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:03.128903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:03.139132Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10365) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:03.158221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518572460652992:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:03.158325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [5:7571518572460652992:2048] [node 1] ICP35 event processing took too much time StateWork:293(18843848ns) {HandleHandshakeStatus:395(51398ns) {LogHandshakeFail:472(13827ns) {UpdateErrorStateLog:544(4632ns)}}}
2025-11-11T17:19:03.035284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:03.086076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:03.086128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518571228282902:2061] [node 4] ICP22 created new session: [6:7571518575523250276:2061]
2025-11-11T17:19:03.086180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518575523250276:2061] [node 4] ICS09 handshake done sender: [6:7571518571228282976:2120] self: [6:7571518571228282978:0] peer: [4:7571518570402942305:0] socket: 248 qp: -1
2025-11-11T17:19:03.086219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518575523250276:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:03.086259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:03.086565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518571228282902:2061] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:03.086909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518571228282885:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.095689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518575523250280:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:03.096525Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518571228282977:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:03.096657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518571228282885:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.096699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:03.096743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518571228282885:2048] [node 1] ICP22 created new session: [6:7571518575523250281:2048]
2025-11-11T17:19:03.096778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518575523250281:2048] [node 1] ICS09 handshake done sender: [6:7571518571228282977:2121] self: [6:7571518575523250278:0] peer: [1:7571518571059938941:0] socket: 254 qp: -1
2025-11-11T17:19:03.096807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518575523250281:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:03.096837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:03.109668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518571228282887:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.146000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518575523250283:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34055
2025-11-11T17:19:03.158091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518571228282887:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(43508973ns) {IncomingHandshake:261(22452725ns) {DropIncomingHandshake:441(197ns) PrepareNewSessionHandshake:476(1424ns) {SwitchToState:178(259ns)} GenerateSessionVirtualId:988(215ns) IssueIncomingHandshakeReply:193(22430504ns) {SwitchToState:178(1687ns)}}}
2025-11-11T17:19:03.165982Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518571228282975:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:03.171239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518571228282887:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.171308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:03.171348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518571228282887:2050] [node 3] ICP22 created new session: [6:7571518575523250284:2050]
2025-11-11T17:19:03.171390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518575523250284:2050] [node 3] ICS09 handshake done sender: [6:7571518571228282975:2119] self: [6:7571518575523250279:0] peer: [3:7571518569926216250:0] socket: 244 qp: -1
2025-11-11T17:19:03.171421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518575523250284:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:03.171476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:03.182350Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24979) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:03.182787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518571228282887:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:03.045874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:03.037691Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518571059938954:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.081633Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518571059938946:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.081675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518571059938946:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:03.094405Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518571059938947:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.094430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518571059938947:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:03.095860Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518571059938946:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:03.096132Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518571059938947:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:03.102276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518571059938912:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.120003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518571059938913:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.120055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:03.137708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518571059938913:2066] [node 6] ICP22 created new session: [1:7571518575354906258:2066]
2025-11-11T17:19:03.157983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:03.159599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518575354906258:2066] [node 6] ICS09 handshake done sender: [1:7571518571059938947:2116] self: [1:7571518571059938941:0] peer: [6:7571518575523250278:0] socket: 252 qp: -1
2025-11-11T17:19:03.159651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518571059938912:2067] [node 5] ICP22 created new session: [1:7571518575354906259:2067]
2025-11-11T17:19:03.165851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518575354906258:2066] [node 6] ICS10 traffic start
2025-11-11T17:19:03.165918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518575354906259:2067] [node 5] ICS09 handshake done sender: [1:7571518571059938946:2115] self: [1:7571518571059938945:0] peer: [5:7571518576755620382:0] socket: 251 qp: -1
2025-11-11T17:19:03.169948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:03.177721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518575354906259:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:03.177786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:03.178251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518571059938912:2067] [node 5] ICP35 event processing took too much time PendingConnection:278(75738729ns) {HandleHandshakeStatus:329(75736435ns) {DropIncomingHandshake:441(12102ns) DropOutgoingHandshake:456(2637ns) SwitchToState:178(742ns) ProcessPendingSessionEvents:495(94071ns) {ForwardSessionEventToSession:612(46720ns) ForwardSessionEventToSession:612(2732ns) ForwardSessionEventToSession:612(9653ns) ForwardSessionEventToSession:612(2682ns) ForwardSessionEventToSession:612(2085ns) ForwardSessionEventToSession:612(9131ns) ForwardSessionEventToSession:612(2390ns) ForwardSessionEventToSession:612(4596ns) ForwardSessionEventToSession:612(2660ns) ForwardSessionEventToSession:612(1726ns)}}}
2025-11-11T17:19:03.186329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518571059938913:2066] [node 6] ICP35 event processing took too much time PendingConnection:278(78824098ns) {HandleHandshakeStatus:329(78820989ns) {DropIncomingHandshake:441(14921ns) DropOutgoingHandshake:456(682ns) SwitchToState:178(505ns) ProcessPendingSessionEvents:495(111868ns) {ForwardSessionEventToSession:612(57504ns) ForwardSessionEventToSession:612(7149ns) ForwardSessionEventToSession:612(3099ns) ForwardSessionEventToSession:612(5949ns) ForwardSessionEventToSession:612(2263ns) ForwardSessionEventToSession:612(3088ns) ForwardSessionEventToSession:612(5330ns) ForwardSessionEventToSession:612(3072ns) ForwardSessionEventToSession:612(2886ns) ForwardSessionEventToSession:612(13012ns)}}}
2025-11-11T17:19:03.059027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518569926216179:2048] [node 1] ICP22 created new session: [3:7571518574221183564:2048]
2025-11-11T17:19:03.045926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518570402942298:2094] [node 6] ICP22 created new session: [4:7571518574697909619:2094]
2025-11-11T17:19:03.266026Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518577618272727:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.266180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518573323305335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.069675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518574221183564:2048] [node 1] ICS09 handshake done sender: [3:7571518569926216254:2115] self: [3:7571518569926216266:0] peer: [1:7571518571059938933:0] socket: 236 qp: -1
2025-11-11T17:19:03.045969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518574697909619:2094] [node 6] ICS09 handshake done sender: [4:7571518570402942306:2116] self: [4:7571518570402942305:0] peer: [6:7571518571228282978:0] socket: 234 qp: -1
2025-11-11T17:19:03.270742Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518575354906252:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.270779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518575354906252:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:03.046005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518574697909619:2094] [node 6] ICS10 traffic start
2025-11-11T17:19:03.046056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:03.058618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518570402942229:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.082097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518570402942241:2060] [node 5] ICP35 event processing took too much time PendingConnection:278(122901640ns) {HandleHandshakeStatus:329(122899596ns) {DropIncomingHandshake:441(206ns) DropOutgoingHandshake:456(267ns) SwitchToState:178(497ns) ProcessPendingSessionEvents:495(47955593ns) {ForwardSessionEventToSession:612(15477275ns) ForwardSessionEventToSession:612(7112ns) ForwardSessionEventToSession:612(5660ns) ForwardSessionEventToSession:612(2962ns) ForwardSessionEventToSession:612(2797ns) ForwardSessionEventToSession:612(8048868ns) ForwardSessionEventToSession:612(8714ns) ForwardSessionEventToSession:612(2288ns) ForwardSessionEventToSession:612(24096676ns) ForwardSessionEventToSession:612(5931ns)}}}
2025-11-11T17:19:03.179267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518574697909622:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65352
2025-11-11T17:19:03.193654Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518570402942307:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:03.221775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518570402942229:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.221835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:03.221883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518570402942229:2050] [node 3] ICP22 created new session: [4:7571518574697909623:2050]
2025-11-11T17:19:03.233734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518574697909623:2050] [node 3] ICS09 handshake done sender: [4:7571518570402942307:2117] self: [4:7571518574697909621:0] peer: [3:7571518569926216256:0] socket: 243 qp: -1
2025-11-11T17:19:03.233778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518574697909623:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:03.233833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:03.069789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518569926216180:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(313558376ns) {Configure:104(313551824ns) {ProcessConfigured:135(17875450ns) {StartInitialHandshake:157(17871741ns) {DropHandshakes:469(3289ns) {DropIncomingHandshake:441(735ns) DropOutgoingHandshake:456(638ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:476(4034ns) {SwitchToState:178(865ns)}}}}}
2025-11-11T17:19:03.069870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518569926216180:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:03.081659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518574221183564:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:03.087316Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216249:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.087348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518569926216249:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:03.087641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:03.105624Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216257:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.105668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518569926216257:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:03.117931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518569926216179:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(80107342ns) {HandleHandshakeStatus:329(80105088ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(23739ns) SwitchToState:178(522ns) ProcessPendingSessionEvents:495(95336ns) {ForwardSessionEventToSession:612(53118ns) ForwardSessionEventToSession:612(2702ns) ForwardSessionEventToSession:612(8160ns) ForwardSessionEventToSession:612(2505ns) ForwardSessionEventToSession:612(2189ns) ForwardSessionEventToSession:612(4740ns) ForwardSessionEventToSession:612(2684ns) ForwardSessionEventToSession:612(4660ns) ForwardSessionEventToSession:612(4034ns) ForwardSessionEventToSession:612(1791ns)}}}
2025-11-11T17:19:03.118057Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518569926216251:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:03.118077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518569926216251:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:03.121836Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518569926216249:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:03.126286Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518569926216251:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:03.126810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518569926216241:2058] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.126898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:03.126941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518569926216241:2058] [node 5] ICP22 created new session: [3:7571518574221183571:2058]
2025-11-11T17:19:03.127002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518574221183571:2058] [node 5] ICS09 handshake done sender: [3:7571518569926216249:2113] self: [3:7571518569926216248:0] peer: [5:7571518576755620383:0] socket: 233 qp: -1
2025-11-11T17:19:03.127035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518574221183571:2058] [node 5] ICS10 traffic start
2025-11-11T17:19:03.127069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:03.127419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518569926216243:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.127471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:03.127512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518569926216243:2065] [node 6] ICP22 created new session: [3:7571518574221183573:2065]
2025-11-11T17:19:03.127561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518574221183573:2065] [node 6] ICS09 handshake done sender: [3:7571518569926216251:2114] self: [3:7571518569926216250:0] peer: [6:7571518575523250279:0] socket: 241 qp: -1
2025-11-11T17:19:03.127588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518574221183573:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:03.127615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:03.138421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518574221183569:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37729
2025-11-11T17:19:03.159174Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518569926216247:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:03.182707Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518569926216257:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:03.201686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518569926216180:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.217693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518569926216244:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.217793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:03.217847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518569926216244:2073] [node 4] ICP22 created new session: [3:7571518574221183576:2073]
2025-11-11T17:19:03.217901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518574221183576:2073] [node 4] ICS09 handshake done sender: [3:7571518569926216257:2117] self: [3:7571518569926216256:0] peer: [4:7571518574697909621:0] socket: 242 qp: -1
2025-11-11T17:19:03.217939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518574221183576:2073] [node 4] ICS10 traffic start
2025-11-11T17:19:03.217992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:03.253678Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:03.262735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:03.262790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518569926216180:2049] [node 2] ICP22 created new session: [3:7571518574221183578:2049]
2025-11-11T17:19:03.270670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518574221183578:2049] [node 2] ICS09 handshake done sender: [3:7571518569926216247:2112] self: [3:7571518574221183566:0] peer: [2:7571518573323305408:0] socket: 228 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:03.274143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518577618272729:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26678
2025-11-11T17:19:03.274881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518575354906252:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:03.273398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518574221183578:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:03.273455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:03.274308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518569926216180:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(84119632ns) {HandleHandshakeStatus:329(84115859ns) {DropIncomingHandshake:441(2739ns) DropOutgoingHandshake:456(6547ns) SwitchToState:178(588ns) ProcessPendingSessionEvents:495(471379ns) {ForwardSessionEventToSession:612(424596ns) ForwardSessionEventToSession:612(8036ns) ForwardSessionEventToSession:612(2494ns) ForwardSessionEventToSession:612(2701ns) ForwardSessionEventToSession:612(7177ns) ForwardSessionEventToSession:612(1923ns) ForwardSessionEventToSession:612(2992ns) ForwardSessionEventToSession:612(4093ns) ForwardSessionEventToSession:612(1877ns) ForwardSessionEventToSession:612(4121ns)}}}
2025-11-11T17:19:03.274364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518569926216180:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:03.278158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518566764971571:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:03.278272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:03.278333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518566764971571:2048] [node 2] ICP22 created new session: [1:7571518575354906264:2048]
2025-11-11T17:19:03.278378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518575354906264:2048] [node 2] ICS09 handshake done sender: [1:7571518575354906252:2118] self: [1:7571518571059938953:0] peer: [2:7571518577618272728:0] socket: 245 qp: -1
2025-11-11T17:19:03.278409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518575354906264:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:03.278478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:03.279055Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518577618272727:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:03.279273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518573323305335:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:03.279328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:03.279371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518573323305335:2048] [node 1] ICP22 created new session: [2:7571518577618272730:2048]
2025-11-11T17:19:03.279406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518577618272730:2048] [node 1] ICS09 handshake done sender: [2:7571518577618272727:2118] self: [2:7571518577618272728:0] peer: [1:7571518571059938953:0] socket: 270 qp: -1
2025-11-11T17:19:03.279432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518577618272730:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:03.279460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:03.389104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:03.391155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:03.391353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518574221183564:2048] [node 1] ICS01 socket: 236 reason# EndOfStream
2025-11-11T17:19:03.391386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518569926216179:2048] [node 1] ICP30 unregister session Session# [3:7571518574221183564:2048] VirtualId# [3:7571518569926216266:0]
2025-11-11T17:19:03.391414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518574221183564:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.391475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:03.391508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:03.389372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518577618272730:2048] [node 1] ICS01 socket: 270 reason# EndOfStream
2025-11-11T17:19:03.389414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518573323305335:2048] [node 1] ICP30 unregister session Session# [2:7571518577618272730:2048] VirtualId# [2:7571518577618272728:0]
2025-11-11T17:19:03.389446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518577618272730:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.389536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:03.389574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:03.401784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:03.402024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518576755620388:2048] [node 1] ICS01 socket: 258 reason# EndOfStream
2025-11-11T17:19:03.402062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518572460652992:2048] [node 1] ICP30 unregister session Session# [5:7571518576755620388:2048] VirtualId# [5:7571518576755620382:0]
2025-11-11T17:19:03.402095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518576755620388:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.402168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:03.402207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:03.405707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:03.405932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518570402942309:2048] [node 1] ICS01 socket: 232 reason# EndOfStream
2025-11-11T17:19:03.405967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518570402942227:2048] [node 1] ICP30 unregister session Session# [4:7571518570402942309:2048] VirtualId# [4:7571518570402942304:0]
2025-11-11T17:19:03.405999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518570402942309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.406097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:03.406139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:03.406232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:03.406458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518575523250281:2048] [node 1] ICS01 socket: 254 reason# EndOfStream
2025-11-11T17:19:03.406500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518571228282885:2048] [node 1] ICP30 unregister session Session# [6:7571518575523250281:2048] VirtualId# [6:7571518575523250278:0]
2025-11-11T17:19:03.406558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518575523250281:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.406620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:03.406654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:03.515313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:03.515545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518574221183578:2049] [node 2] ICS01 socket: 228 reason# EndOfStream
2025-11-11T17:19:03.515586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518569926216180:2049] [node 2] ICP30 unregister session Session# [3:7571518574221183578:2049] VirtualId# [3:7571518574221183566:0]
2025-11-11T17:19:03.515614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518574221183578:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.515673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:03.515716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:03.521722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:03.521958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518570402942318:2049] [node 2] ICS01 socket: 221 reason# EndOfStream
2025-11-11T17:19:03.521989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518570402942228:2049] [node 2] ICP30 unregister session Session# [4:7571518570402942318:2049] VirtualId# [4:7571518570402942315:0]
2025-11-11T17:19:03.522019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518570402942318:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.522121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:03.522159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:03.523265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:03.523472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518571228282969:2049] [node 2] ICS01 socket: 216 reason# EndOfStream
2025-11-11T17:19:03.523497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518571228282886:2049] [node 2] ICP30 unregister session Session# [6:7571518571228282969:2049] VirtualId# [6:7571518571228282961:0]
2025-11-11T17:19:03.523520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518571228282969:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.523589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:03.523616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:03.525678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:03.525924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518576755620380:2049] [node 2] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-11T17:19:03.525970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518576755620380:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.526096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:03.526166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518576755620380:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:03.527012Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Couldn't connect to any resolved address
2025-11-11T17:19:03.527276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518572460652993:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:03.527319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518576755620380:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:03.527367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518572460652993:2049] [node 2] ICP30 unregister session Session# [5:7571518576755620380:2049] VirtualId# [5:7571518572460653080:0]
2025-11-11T17:19:03.527409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:03.527435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518572460652993:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:03.527450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13928) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:03.537694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518572460652993:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:03.605790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:03.606023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518575523250284:2050] [node 3] ICS01 socket: 244 reason# EndOfStream
2025-11-11T17:19:03.606056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518571228282887:2050] [node 3] ICP30 unregister session Session# [6:7571518575523250284:2050] VirtualId# [6:7571518575523250279:0]
2025-11-11T17:19:03.606084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518575523250284:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.606158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:03.606198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:03.606280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:03.606480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518574697909623:2050] [node 3] ICS07 socket disconnect 243 reason# EndOfStream
2025-11-11T17:19:03.606510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518574697909623:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.606566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:03.606600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518574697909623:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:03.607906Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24979) Couldn't connect to any resolved address
2025-11-11T17:19:03.608143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518570402942229:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24979) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:03.608185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518574697909623:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:03.608212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518570402942229:2050] [node 3] ICP30 unregister session Session# [4:7571518574697909623:2050] VirtualId# [4:7571518574697909621:0]
2025-11-11T17:19:03.608252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:03.608279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518570402942229:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24979) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:03.608294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24979) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:03.609681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:03.609877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518576755620390:2050] [node 3] ICS01 socket: 237 reason# EndOfStream
2025-11-11T17:19:03.609910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518572460652994:2050] [node 3] ICP30 unregister session Session# [5:7571518576755620390:2050] VirtualId# [5:7571518576755620383:0]
2025-11-11T17:19:03.609935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518576755620390:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.610010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:03.610052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:03.621678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518570402942229:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:03.689770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:03.690001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518575523250276:2061] [node 4] ICS01 socket: 248 reason# EndOfStream
2025-11-11T17:19:03.690041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518571228282902:2061] [node 4] ICP30 unregister session Session# [6:7571518575523250276:2061] VirtualId# [6:7571518571228282978:0]
2025-11-11T17:19:03.690071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518575523250276:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.690152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:03.690198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:03.693700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:03.693926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518576755620384:2061] [node 4] ICS01 socket: 201 reason# EndOfStream
2025-11-11T17:19:03.693956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518572460653012:2061] [node 4] ICP30 unregister session Session# [5:7571518576755620384:2061] VirtualId# [5:7571518572460653079:0]
2025-11-11T17:19:03.693984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518576755620384:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.694059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:03.694096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:03.769762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:03.769996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518571228282973:2066] [node 5] ICS01 socket: 222 reason# EndOfStream
2025-11-11T17:19:03.770031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518571228282910:2066] [node 5] ICP30 unregister session Session# [6:7571518571228282973:2066] VirtualId# [6:7571518571228282964:0]
2025-11-11T17:19:03.770067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518571228282973:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:03.770177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:03.770221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:03.965830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518575020189077:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:03.965872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518575020189078:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:04.033881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518578856952386:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:04.033926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518578856952387:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:04.093294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518579906148770:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:04.093334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518579906148771:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:04.156123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518579970649589:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:04.156170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518579970649590:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:04.156195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518579970649591:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:04.221918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518579986289761:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:04.221961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518579986289762:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:04.221987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518579986289763:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:04.301926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518578874438622:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:04.301969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518578874438623:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:04.301995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518578874438624:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:04.651751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518579986289762:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(40307644ns) {Configure:104(40301640ns) {ProcessConfigured:135(93304ns) {StartInitialHandshake:157(90276ns) {DropHandshakes:469(3798ns) {DropIncomingHandshake:441(1158ns) DropOutgoingHandshake:456(561ns)} GenerateSessionVirtualId:988(1054ns) PrepareNewSessionHandshake:476(2982ns) {SwitchToState:178(1423ns)}}}}}
2025-11-11T17:19:04.613761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518578856952417:2070] [node 4] ICP01 ready to work
2025-11-11T17:19:04.641757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518579970649614:2068] [node 5] ICP01 ready to work
2025-11-11T17:19:04.640608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518579315156428:2092] [node 5] ICP01 ready to work
2025-11-11T17:19:04.751530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518579986289790:2070] [node 4] ICP01 ready to work
2025-11-11T17:19:04.754888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518579906148797:2058] [node 6] ICP01 ready to work
2025-11-11T17:19:04.673672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518579315156427:2088] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.785620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518578856952427:2068] [node 6] ICP01 ready to work
2025-11-11T17:19:04.737700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518579986289781:2061] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.782036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518579970649641:2062] [node 6] ICP01 ready to work
2025-11-11T17:19:04.785632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518579906148793:2069] [node 5] ICP01 ready to work
2025-11-11T17:19:04.931083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518579970649589:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(173113782ns) {Configure:104(173110593ns) {ProcessConfigured:135(79076ns) {StartInitialHandshake:157(75420ns) {DropHandshakes:469(8380ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(542ns)} GenerateSessionVirtualId:988(639ns) PrepareNewSessionHandshake:476(1104ns) {SwitchToState:178(315ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.757747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518579315156436:2090] [node 4] ICP01 ready to work
2025-11-11T17:19:04.943243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518579970649591:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(301314243ns) {Configure:104(301308495ns) {ProcessConfigured:135(61647ns) {StartInitialHandshake:157(59345ns) {DropHandshakes:469(2848ns) {DropIncomingHandshake:441(854ns) DropOutgoingHandshake:456(520ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:476(1934ns) {SwitchToState:178(675ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.853927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518578856952437:2077] [node 5] ICP01 ready to work
2025-11-11T17:19:04.877812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518578874438681:2059] [node 5] ICP01 ready to work
2025-11-11T17:19:04.869104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518579986289826:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.867885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518579906148802:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53290
2025-11-11T17:19:04.803235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518575020189078:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(217218756ns) {Configure:104(217214162ns) {ProcessConfigured:135(57566ns) {StartInitialHandshake:157(53071ns) {DropHandshakes:469(3021ns) {DropIncomingHandshake:441(789ns) DropOutgoingHandshake:456(633ns)} GenerateSessionVirtualId:988(1031ns) PrepareNewSessionHandshake:476(2024ns) {SwitchToState:178(640ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.884917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518578856952450:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49398
2025-11-11T17:19:04.951875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518578874438695:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.868064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518579906148803:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53290
2025-11-11T17:19:04.959067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518579986289781:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(133184174ns) {Configure:104(133177824ns) {ProcessConfigured:135(74303ns) {StartInitialHandshake:157(71900ns) {DropHandshakes:469(2821ns) {DropIncomingHandshake:441(557ns) DropOutgoingHandshake:456(471ns)} GenerateSessionVirtualId:988(1021ns) PrepareNewSessionHandshake:476(2007ns) {SwitchToState:178(579ns)}}}}}
2025-11-11T17:19:04.932102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518579315156442:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29030
2025-11-11T17:19:04.868196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518579906148824:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53290
2025-11-11T17:19:04.873745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518579906148803:2078] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:04.880549Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518579906148824:2098] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:04.941608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518579906148827:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:05.001896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518579906148770:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(396604995ns) {Configure:104(396600234ns) {ProcessConfigured:135(23320255ns) {StartInitialHandshake:157(23317496ns) {DropHandshakes:469(1707ns) {DropIncomingHandshake:441(278ns) DropOutgoingHandshake:456(461ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:476(4123ns) {SwitchToState:178(1650ns)}}}}}
2025-11-11T17:19:05.013645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518579906148802:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.973930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518578874438696:2068] [node 4] ICP01 ready to work
2025-11-11T17:19:04.885103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518578856952451:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49398
2025-11-11T17:19:04.959714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518579986289790:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(206746066ns) {Configure:104(206741701ns) {ProcessConfigured:135(20703200ns) {StartInitialHandshake:157(20701382ns) {DropHandshakes:469(3169ns) {DropIncomingHandshake:441(920ns) DropOutgoingHandshake:456(506ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:476(1063ns) {SwitchToState:178(138ns)}}}}}
2025-11-11T17:19:04.976158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518579986289837:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:05.049758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518579986289826:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.049893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518579986289763:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.040809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518579906148770:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.041155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518584201116142:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53290
2025-11-11T17:19:05.070261Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518579906148823:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.070289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518579906148823:2097] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.015035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518575020189077:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(373171930ns) {Configure:104(373163059ns) {ProcessConfigured:135(75902ns) {StartInitialHandshake:157(72829ns) {DropHandshakes:469(3313ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(583ns)} GenerateSessionVirtualId:988(1397ns) PrepareNewSessionHandshake:476(1878ns) {SwitchToState:178(605ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.976598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518578874438695:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:04.965663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518578856952450:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.078177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518579906148797:2058] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(271327570ns) {Configure:104(271322857ns) {ProcessConfigured:135(12262486ns) {StartInitialHandshake:157(12259881ns) {DropHandshakes:469(9336ns) {DropIncomingHandshake:441(875ns) DropOutgoingHandshake:456(442ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:476(2228ns) {SwitchToState:178(480ns)}}}}}
2025-11-11T17:19:05.078603Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518584201116142:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.093801Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518579906148823:2097] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:05.094060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518579906148793:2069] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.113935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.116235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616956:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
2025-11-11T17:19:05.026815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518579315156427:2088] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(268949072ns) {Configure:104(268941187ns) {ProcessConfigured:135(62814ns) {StartInitialHandshake:157(60428ns) {DropHandshakes:469(2735ns) {DropIncomingHandshake:441(655ns) DropOutgoingHandshake:456(523ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:476(2770ns) {SwitchToState:178(1341ns)}}}}}
2025-11-11T17:19:04.976697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518578874438623:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.133937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518579906148793:2069] [node 5] ICP22 created new session: [3:7571518584201116150:2069]
2025-11-11T17:19:05.142048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518579970649590:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(527983725ns) {Configure:104(527980385ns) {ProcessConfigured:135(62676ns) {StartInitialHandshake:157(60108ns) {DropHandshakes:469(1748ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(301ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(1596ns) {SwitchToState:178(525ns)}}}}}
2025-11-11T17:19:04.976927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518578856952453:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.105877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518579986289761:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(507906193ns) {Configure:104(507898746ns) {ProcessConfigured:135(27994577ns) {StartInitialHandshake:157(27991418ns) {DropHandshakes:469(3672ns) {DropIncomingHandshake:441(951ns) DropOutgoingHandshake:456(962ns)} GenerateSessionVirtualId:988(726ns) PrepareNewSessionHandshake:476(3117ns) {SwitchToState:178(887ns)}}}}}
2025-11-11T17:19:05.049917Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518583610123740:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:04.986037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518578874438700:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.153704Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518584265616956:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.110380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518584281257138:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:04.976949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518578856952453:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.165887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518579970649614:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(431250136ns) {Configure:104(431241368ns) {ProcessConfigured:135(11752746ns) {StartInitialHandshake:157(11749478ns) {DropHandshakes:469(3929ns) {DropIncomingHandshake:441(1137ns) DropOutgoingHandshake:456(530ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:476(4165ns) {SwitchToState:178(2369ns)}}}}}
2025-11-11T17:19:05.166660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616961:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.135290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518584201116150:2069] [node 5] ICS09 handshake done sender: [3:7571518579906148823:2097] self: [3:7571518579906148822:0] peer: [5:7571518584281257135:0] socket: 219 qp: -1
2025-11-11T17:19:05.049941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518583610123740:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.986091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518578874438701:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.977664Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518578856952453:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:05.055609Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518583610123740:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:05.142489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518579906148771:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(500355443ns) {Configure:104(468331788ns) {ProcessConfigured:135(8389504ns) {StartInitialHandshake:157(8387764ns) {DropHandshakes:469(3638ns) {DropIncomingHandshake:441(686ns) DropOutgoingHandshake:456(558ns)} GenerateSessionVirtualId:988(650ns) PrepareNewSessionHandshake:476(1531ns) {SwitchToState:178(100ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.982206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518578856952427:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.065956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518578874438622:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(308059043ns) {Configure:104(308056019ns) {ProcessConfigured:135(59024ns) {StartInitialHandshake:157(56912ns) {DropHandshakes:469(2611ns) {DropIncomingHandshake:441(633ns) DropOutgoingHandshake:456(408ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:476(1956ns) {SwitchToState:178(923ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.229997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518584265616961:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.253926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518579970649641:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(367493688ns) {Configure:104(367486167ns) {ProcessConfigured:135(23771028ns) {StartInitialHandshake:157(23767539ns) {DropHandshakes:469(3060ns) {DropIncomingHandshake:441(410ns) DropOutgoingHandshake:456(621ns)} GenerateSessionVirtualId:988(806ns) PrepareNewSessionHandshake:476(4705ns) {SwitchToState:178(2360ns)}}}}}
2025-11-11T17:19:05.197676Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518579986289826:2107] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:05.142849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518579906148771:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.079614Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518578874438695:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.056003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518583610123746:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29030
2025-11-11T17:19:05.056105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518583610123747:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29030
2025-11-11T17:19:04.982252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:05.143700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518584201116150:2069] [node 5] ICS10 traffic start
2025-11-11T17:19:05.197720Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10960) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:05.079749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518583169406000:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.079832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518583169406001:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.114140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518578874438623:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.114205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:05.114245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518578874438623:2049] [node 2] ICP22 created new session: [6:7571518583169406002:2049]
2025-11-11T17:19:05.114291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518583169406002:2049] [node 2] ICS09 handshake done sender: [6:7571518578874438695:2113] self: [6:7571518578874438699:0] peer: [2:7571518578856952452:0] socket: 225 qp: -1
2025-11-11T17:19:05.114327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518583169406002:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:05.114361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:05.134093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518578874438700:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.134211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518578874438681:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.261899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518579970649590:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:04.982295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518578856952427:2068] [node 6] ICP22 created new session: [2:7571518578856952457:2068]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.056428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518575020189077:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.143753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:05.198372Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518579986289837:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.135007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518583169406000:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.149015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518583169406001:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.156176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518578874438622:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.167558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518578874438624:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.170679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518583169406008:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.171801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518583169406009:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.181878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518578874438622:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(21111389ns) {IncomingHandshake:261(21109650ns) {DropIncomingHandshake:441(181ns) PrepareNewSessionHandshake:476(590ns) {SwitchToState:178(253ns)} GenerateSessionVirtualId:988(219ns) IssueIncomingHandshakeReply:193(8958901ns) {SwitchToState:178(1580ns)}}}
2025-11-11T17:19:05.186672Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518583169406000:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:05.186882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518578874438622:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.186932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:05.186972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518578874438622:2048] [node 1] ICP22 created new session: [6:7571518583169406010:2048]
2025-11-11T17:19:05.187015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518583169406010:2048] [node 1] ICS09 handshake done sender: [6:7571518583169406000:2118] self: [6:7571518583169406006:0] peer: [1:7571518583610123742:0] socket: 246 qp: -1
2025-11-11T17:19:05.187048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518583169406010:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:05.187086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:05.221667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518578874438624:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18351164ns) {IncomingHandshake:261(18349096ns) {DropIncomingHandshake:441(537ns) PrepareNewSessionHandshake:476(1708ns) {SwitchToState:178(834ns)} GenerateSessionVirtualId:988(466ns) IssueIncomingHandshakeReply:193(12529633ns) {SwitchToState:178(1188ns)}}}
2025-11-11T17:19:05.242286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518583169406012:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.245668Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518583169406001:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:05.289925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518578874438624:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.293872Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10960) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:05.305816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:05.313902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518578874438624:2050] [node 3] ICP22 created new session: [6:7571518583169406013:2050]
2025-11-11T17:19:05.314631Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518578874438700:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:05.317987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518578874438681:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.318052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:05.318098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518578874438681:2059] [node 5] ICP22 created new session: [6:7571518583169406014:2059]
2025-11-11T17:19:05.318143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518583169406014:2059] [node 5] ICS09 handshake done sender: [6:7571518578874438700:2115] self: [6:7571518583169406005:0] peer: [5:7571518579986289832:0] socket: 230 qp: -1
2025-11-11T17:19:05.318178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518583169406014:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:05.318213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:04.982358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518578856952457:2068] [node 6] ICS09 handshake done sender: [2:7571518578856952453:2110] self: [2:7571518578856952452:0] peer: [6:7571518578874438699:0] socket: 222 qp: -1
2025-11-11T17:19:05.262059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518579970649590:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16062559ns) {IncomingHandshake:261(16060260ns) {DropIncomingHandshake:441(684ns) PrepareNewSessionHandshake:476(2396ns) {SwitchToState:178(1126ns)} GenerateSessionVirtualId:988(914ns) IssueIncomingHandshakeReply:193(60447ns) {SwitchToState:178(1102ns)}}}
2025-11-11T17:19:05.157937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518584201116157:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53290
2025-11-11T17:19:05.209615Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518579986289833:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.056469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:05.334606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518583169406013:2050] [node 3] ICS09 handshake done sender: [6:7571518583169406001:2119] self: [6:7571518583169406007:0] peer: [3:7571518584201116145:0] socket: 247 qp: -1
2025-11-11T17:19:04.982399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518578856952457:2068] [node 6] ICS10 traffic start
2025-11-11T17:19:05.209658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518579986289833:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.213294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518579986289763:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10960) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518579986289826:2107] held: no
2025-11-11T17:19:05.213375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518579986289763:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.213405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:05.213443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518579986289763:2050] [node 3] ICP22 created new session: [5:7571518584281257140:2050]
2025-11-11T17:19:05.213475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518584281257140:2050] [node 3] ICS09 handshake done sender: [5:7571518579986289826:2107] self: [5:7571518584281257135:0] peer: [3:7571518579906148822:0] socket: 220 qp: -1
2025-11-11T17:19:05.213494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518584281257140:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:05.213519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:05.269841Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518579986289833:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:05.293630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518584281257139:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:05.298505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518584281257144:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:05.306004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518579986289781:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.313946Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518584281257144:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.314389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518579986289761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.315125Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518584281257139:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.334229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518584281257149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:05.337652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518579986289762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.341658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:05.346513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518579986289781:2061] [node 6] ICP22 created new session: [5:7571518584281257150:2061]
2025-11-11T17:19:05.298113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.166255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518579906148827:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(164022140ns) {Configure:104(164017216ns) {ProcessConfigured:135(12100004ns) {StartInitialHandshake:157(12097839ns) {DropHandshakes:469(541ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(84ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(1638ns) {SwitchToState:178(649ns)}}}}}
2025-11-11T17:19:05.056513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518575020189077:2048] [node 2] ICP22 created new session: [1:7571518583610123748:2048]
2025-11-11T17:19:04.982437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:05.338915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518583169406013:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.374051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518584281257151:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:05.298649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616969:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.170864Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518584201116146:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.056554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518583610123748:2048] [node 2] ICS09 handshake done sender: [1:7571518583610123740:2112] self: [1:7571518583610123739:0] peer: [2:7571518583151919762:0] socket: 236 qp: -1
2025-11-11T17:19:05.374378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518584281257150:2061] [node 6] ICS09 handshake done sender: [5:7571518579986289833:2112] self: [5:7571518579986289832:0] peer: [6:7571518583169406005:0] socket: 228 qp: -1
2025-11-11T17:19:04.998688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518578856952387:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(400713565ns) {Configure:104(400709441ns) {ProcessConfigured:135(89652ns) {StartInitialHandshake:157(86733ns) {DropHandshakes:469(2915ns) {DropIncomingHandshake:441(692ns) DropOutgoingHandshake:456(614ns)} GenerateSessionVirtualId:988(1407ns) PrepareNewSessionHandshake:476(4326ns) {SwitchToState:178(548ns)}}}}}
2025-11-11T17:19:05.299357Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518584265616961:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:05.346958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:05.056582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518583610123748:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:05.056616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:05.069828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518583610123747:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.383338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518584281257150:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:05.015771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518578856952462:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49398
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.382106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518578874438624:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(91973492ns) {HandleHandshakeStatus:329(91971137ns) {DropIncomingHandshake:441(660ns) DropOutgoingHandshake:456(3805ns) SwitchToState:178(865ns) ProcessPendingSessionEvents:495(113951ns) {ForwardSessionEventToSession:612(64409ns) ForwardSessionEventToSession:612(2739ns) ForwardSessionEventToSession:612(2673ns) ForwardSessionEventToSession:612(10293ns) ForwardSessionEventToSession:612(1916ns) ForwardSessionEventToSession:612(7451ns) ForwardSessionEventToSession:612(2368ns) ForwardSessionEventToSession:612(2622ns) ForwardSessionEventToSession:612(4501ns) ForwardSessionEventToSession:612(3355ns)}}}
2025-11-11T17:19:05.299528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518579970649590:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.170884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518584201116146:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.073773Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518579315156440:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.073815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518579315156440:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.089651Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518583610123746:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.110890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518575020189077:2048] [node 2] ICP18 handshake (actor [1:7571518583610123747:2115]) is too old
2025-11-11T17:19:05.110961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [1:7571518575020189077:2048] [node 2] ICP35 event processing took too much time StateWork:293(40984760ns) {IncomingHandshake:261(40981860ns)}
2025-11-11T17:19:05.154108Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518579315156442:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.169752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518583610123743:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.169776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518583610123743:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.177832Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518579315156440:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:05.185661Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518583610123743:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:05.210672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518575020189078:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.210740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:05.210789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518575020189078:2049] [node 3] ICP22 created new session: [1:7571518583610123756:2049]
2025-11-11T17:19:05.210836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518583610123756:2049] [node 3] ICS09 handshake done sender: [1:7571518579315156440:2110] self: [1:7571518579315156439:0] peer: [3:7571518584201116143:0] socket: 216 qp: -1
2025-11-11T17:19:05.210872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518583610123756:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:05.210934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:05.211456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518579315156427:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.211497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:05.211536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518579315156427:2088] [node 6] ICP22 created new session: [1:7571518583610123758:2088]
2025-11-11T17:19:05.211579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518583610123758:2088] [node 6] ICS09 handshake done sender: [1:7571518583610123743:2113] self: [1:7571518583610123742:0] peer: [6:7571518583169406006:0] socket: 238 qp: -1
2025-11-11T17:19:05.211605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518583610123758:2088] [node 6] ICS10 traffic start
2025-11-11T17:19:05.211636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:05.273856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518579315156428:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(460168745ns) {Configure:104(460162813ns) {ProcessConfigured:135(30395214ns) {StartInitialHandshake:157(30392533ns) {DropHandshakes:469(3612ns) {DropIncomingHandshake:441(859ns) DropOutgoingHandshake:456(630ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:476(2003ns) {SwitchToState:178(434ns)}}}}}
2025-11-11T17:19:05.301970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518579315156436:2090] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(387807406ns) {Configure:104(387799915ns) {ProcessConfigured:135(28154607ns) {StartInitialHandshake:157(28151171ns) {DropHandshakes:469(1877ns) {DropIncomingHandshake:441(375ns) DropOutgoingHandshake:456(166ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:476(3503ns) {SwitchToState:178(1349ns)}}}}}
2025-11-11T17:19:05.337611Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518583610123755:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.337644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518583610123755:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.361769Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518583610123755:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:05.410167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518579315156428:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.416387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:05.434238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518579315156428:2092] [node 5] ICP22 created new session: [1:7571518583610123766:2092]
2025-11-11T17:19:05.190311Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518584201116146:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:05.201627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518584201116141:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53290
2025-11-11T17:19:05.201836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518584201116160:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53290
2025-11-11T17:19:05.221656Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518584201116142:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:05.241839Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518579906148824:2098] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:05.254339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518579906148797:2058] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.254408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:05.254459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518579906148797:2058] [node 6] ICP22 created new session: [3:7571518584201116161:2058]
2025-11-11T17:19:05.254508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518584201116161:2058] [node 6] ICS09 handshake done sender: [3:7571518584201116146:2117] self: [3:7571518584201116145:0] peer: [6:7571518583169406007:0] socket: 245 qp: -1
2025-11-11T17:19:05.254550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518584201116161:2058] [node 6] ICS10 traffic start
2025-11-11T17:19:05.254590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:05.255151Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518584201116141:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.255239Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:05.281840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518579906148771:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518584201116142:2116] held: no
2025-11-11T17:19:05.281930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518579906148771:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.281961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:05.282004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518579906148771:2049] [node 2] ICP22 created new session: [3:7571518584201116164:2049]
2025-11-11T17:19:05.282045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518584201116164:2049] [node 2] ICS09 handshake done sender: [3:7571518584201116142:2116] self: [3:7571518584201116153:0] peer: [2:7571518578856952459:0] socket: 235 qp: -1
2025-11-11T17:19:05.282084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518584201116164:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:05.282115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:05.282495Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:05.283228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518579906148770:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.283271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:05.283309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518579906148770:2048] [node 1] ICP22 created new session: [3:7571518584201116166:2048]
2025-11-11T17:19:05.283364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518584201116166:2048] [node 1] ICS09 handshake done sender: [3:7571518579906148824:2098] self: [3:7571518584201116143:0] peer: [1:7571518579315156439:0] socket: 218 qp: -1
2025-11-11T17:19:05.283395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518584201116166:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:05.283423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:05.283633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518579906148770:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:05.328657Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518584201116158:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.328678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518584201116158:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.333648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518579906148793:2069] [node 5] ICP35 event processing took too much time PendingConnection:278(179690467ns) {HandleHandshakeStatus:329(179687982ns) {DropIncomingHandshake:441(12376ns) DropOutgoingHandshake:456(1873ns) SwitchToState:178(822ns) ProcessPendingSessionEvents:495(78750184ns) {ForwardSessionEventToSession:612(10889205ns) ForwardSessionEventToSession:612(23943851ns) ForwardSessionEventToSession:612(13442575ns) ForwardSessionEventToSession:612(3682ns) ForwardSessionEventToSession:612(2752ns) ForwardSessionEventToSession:612(14606075ns) ForwardSessionEventToSession:612(3288ns) ForwardSessionEventToSession:612(4540764ns) ForwardSessionEventToSession:612(4638ns) ForwardSessionEventToSession:612(11292116ns)}}}
2025-11-11T17:19:05.338992Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518584201116158:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:05.377902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518579906148827:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.385819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:05.385870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518579906148827:2064] [node 4] ICP22 created new session: [3:7571518584201116170:2064]
2025-11-11T17:19:05.385924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518584201116170:2064] [node 4] ICS09 handshake done sender: [3:7571518584201116158:2120] self: [3:7571518584201116156:0] peer: [4:7571518584265616974:0] socket: 260 qp: -1
2025-11-11T17:19:05.385953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518584201116170:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:05.385996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.032959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518578856952462:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.386910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:05.398306Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518584281257149:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.398384Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518584281257144:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:05.401334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518579986289761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.401368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:05.401404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518579986289761:2048] [node 1] ICP22 created new session: [5:7571518584281257154:2048]
2025-11-11T17:19:05.401438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518584281257154:2048] [node 1] ICS09 handshake done sender: [5:7571518584281257144:2119] self: [5:7571518584281257147:0] peer: [1:7571518583610123752:0] socket: 212 qp: -1
2025-11-11T17:19:05.401466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518584281257154:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:05.401493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:05.402916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518579986289790:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.417705Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:05.422523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518579986289762:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518584281257139:2118] held: no
2025-11-11T17:19:05.430209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518579986289781:2061] [node 6] ICP35 event processing took too much time PendingConnection:278(124002854ns) {HandleHandshakeStatus:329(124001090ns) {DropIncomingHandshake:441(14205ns) DropOutgoingHandshake:456(1951ns) SwitchToState:178(458ns) ProcessPendingSessionEvents:495(7039442ns) {ForwardSessionEventToSession:612(6983944ns) ForwardSessionEventToSession:612(5774ns) ForwardSessionEventToSession:612(7295ns) ForwardSessionEventToSession:612(2786ns) ForwardSessionEventToSession:612(2866ns) ForwardSessionEventToSession:612(4494ns) ForwardSessionEventToSession:612(5478ns) ForwardSessionEventToSession:612(4632ns) ForwardSessionEventToSession:612(2519ns) ForwardSessionEventToSession:612(2583ns)}}}
2025-11-11T17:19:05.460503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518584281257157:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:05.457694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518583610123766:2092] [node 5] ICS09 handshake done sender: [1:7571518583610123755:2116] self: [1:7571518583610123752:0] peer: [5:7571518584281257147:0] socket: 264 qp: -1
2025-11-11T17:19:05.457760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518583610123766:2092] [node 5] ICS10 traffic start
2025-11-11T17:19:05.457816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:05.461834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518583610123764:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29030
2025-11-11T17:19:05.382170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518578874438624:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:05.299577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:05.393928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518583169406017:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.400952Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518583169406017:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.401098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518578874438696:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.407197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518583169406019:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22560
2025-11-11T17:19:05.407855Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518583169406017:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:05.408069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518578874438696:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.408141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:05.408190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518578874438696:2068] [node 4] ICP22 created new session: [6:7571518583169406020:2068]
2025-11-11T17:19:05.408230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518583169406020:2068] [node 4] ICS09 handshake done sender: [6:7571518583169406017:2123] self: [6:7571518583169406018:0] peer: [4:7571518584265616963:0] socket: 273 qp: -1
2025-11-11T17:19:05.408260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518583169406020:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:05.408293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:05.299616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518579970649590:2049] [node 2] ICP22 created new session: [4:7571518584265616970:2049]
2025-11-11T17:19:05.299652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518584265616970:2049] [node 2] ICS09 handshake done sender: [4:7571518584265616961:2115] self: [4:7571518584265616964:0] peer: [2:7571518583151919767:0] socket: 252 qp: -1
2025-11-11T17:19:05.299689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518584265616970:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:05.299733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:05.310109Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:05.317638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518579970649590:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:05.322339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616972:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
2025-11-11T17:19:05.328258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518584265616969:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.328376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518579970649591:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.329464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616975:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
2025-11-11T17:19:05.349852Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518584265616969:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:05.354748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518579970649591:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.403135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:05.406005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518584265616965:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.406026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518584265616965:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.407430Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518584265616965:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:05.409672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518579970649641:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.410049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518579970649591:2050] [node 3] ICP22 created new session: [4:7571518584265616977:2050]
2025-11-11T17:19:05.410080Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518584265616972:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.422781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:05.429663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518579970649641:2062] [node 6] ICP22 created new session: [4:7571518584265616979:2062]
2025-11-11T17:19:05.430814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616967:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
2025-11-11T17:19:05.433633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518584265616977:2050] [node 3] ICS09 handshake done sender: [4:7571518584265616969:2119] self: [4:7571518584265616974:0] peer: [3:7571518584201116156:0] socket: 213 qp: -1
2025-11-11T17:19:05.433684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518584265616977:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:05.438069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518584265616979:2062] [node 6] ICS09 handshake done sender: [4:7571518584265616965:2116] self: [4:7571518584265616963:0] peer: [6:7571518583169406018:0] socket: 272 qp: -1
2025-11-11T17:19:05.438114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518584265616979:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:05.438167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:05.438561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518579970649641:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(28695164ns) {HandleHandshakeStatus:329(28693411ns) {DropIncomingHandshake:441(28746ns) DropOutgoingHandshake:456(389ns) SwitchToState:178(353ns) ProcessPendingSessionEvents:495(86130ns) {ForwardSessionEventToSession:612(47598ns) ForwardSessionEventToSession:612(2651ns) ForwardSessionEventToSession:612(4572ns) ForwardSessionEventToSession:612(2771ns) ForwardSessionEventToSession:612(1844ns) ForwardSessionEventToSession:612(4910ns) ForwardSessionEventToSession:612(2218ns) ForwardSessionEventToSession:612(6892ns) ForwardSessionEventToSession:612(2334ns) ForwardSessionEventToSession:612(2676ns)}}}
2025-11-11T17:19:05.445861Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518584265616960:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.445892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518584265616960:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.455252Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518584265616967:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.457680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:05.458983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518579970649589:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.478455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518579970649591:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(125956445ns) {HandleHandshakeStatus:329(125953159ns) {DropIncomingHandshake:441(2340ns) DropOutgoingHandshake:456(6896ns) SwitchToState:178(611ns) ProcessPendingSessionEvents:495(8260346ns) {ForwardSessionEventToSession:612(8197224ns) ForwardSessionEventToSession:612(17669ns) ForwardSessionEventToSession:612(2343ns) ForwardSessionEventToSession:612(3004ns) ForwardSessionEventToSession:612(5442ns) ForwardSessionEventToSession:612(2862ns) ForwardSessionEventToSession:612(7687ns) ForwardSessionEventToSession:612(3166ns) ForwardSessionEventToSession:612(2514ns) ForwardSessionEventToSession:612(7017ns)}}}
2025-11-11T17:19:05.478637Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518584265616960:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.480099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518584281257153:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12887
2025-11-11T17:19:05.046101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518578856952386:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(432243891ns) {Configure:104(432238166ns) {ProcessConfigured:135(56956ns) {StartInitialHandshake:157(54137ns) {DropHandshakes:469(1607ns) {DropIncomingHandshake:441(406ns) DropOutgoingHandshake:456(252ns)} GenerateSessionVirtualId:988(689ns) PrepareNewSessionHandshake:476(2179ns) {SwitchToState:178(720ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:05.499893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518584265616985:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5981
2025-11-11T17:19:05.498520Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518584281257149:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:05.477876Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:05.046266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518578856952386:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:05.503606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518579970649614:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.507884Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:05.513950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:05.517866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518579970649589:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518584265616967:2117] held: no
2025-11-11T17:19:05.506033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518579986289790:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.506103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:05.506145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518579986289790:2070] [node 4] ICP22 created new session: [5:7571518584281257158:2070]
2025-11-11T17:19:05.506197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518584281257158:2070] [node 4] ICS09 handshake done sender: [5:7571518584281257149:2120] self: [5:7571518584281257156:0] peer: [4:7571518584265616959:0] socket: 227 qp: -1
2025-11-11T17:19:05.506230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518584281257158:2070] [node 4] ICS10 traffic start
2025-11-11T17:19:05.506262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:05.513679Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518584281257139:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:05.051277Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518578856952451:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.480251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518583610123760:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.480283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518583610123760:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.482241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518579315156428:2092] [node 5] ICP35 event processing took too much time PendingConnection:278(78625600ns) {HandleHandshakeStatus:329(71831222ns) {DropIncomingHandshake:441(209ns) DropOutgoingHandshake:456(408ns) SwitchToState:178(1612ns) ProcessPendingSessionEvents:495(24068432ns) {ForwardSessionEventToSession:612(52630ns) ForwardSessionEventToSession:612(3266ns) ForwardSessionEventToSession:612(3906496ns) ForwardSessionEventToSession:612(2828ns) ForwardSessionEventToSession:612(2926ns) ForwardSessionEventToSession:612(7773339ns) ForwardSessionEventToSession:612(6503ns) ForwardSessionEventToSession:612(467869ns) ForwardSessionEventToSession:612(11818604ns) ForwardSessionEventToSession:612(3129ns)}}}
2025-11-11T17:19:05.482624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518579315156428:2092] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:05.500183Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518583610123760:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:05.500486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518579315156436:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.500548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:05.500596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518579315156436:2090] [node 4] ICP22 created new session: [1:7571518583610123769:2090]
2025-11-11T17:19:05.500637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518583610123769:2090] [node 4] ICS09 handshake done sender: [1:7571518583610123760:2117] self: [1:7571518583610123754:0] peer: [4:7571518584265616983:0] socket: 265 qp: -1
2025-11-11T17:19:05.500671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518583610123769:2090] [node 4] ICS10 traffic start
2025-11-11T17:19:05.500719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:05.518372Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518584265616967:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:05.532570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518579986289762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.053910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518578856952427:2068] [node 6] ICP18 handshake (actor [2:7571518578856952451:2109]) is too old
2025-11-11T17:19:05.055439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518583151919764:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49398
2025-11-11T17:19:05.057611Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518578856952462:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:05.062038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518578856952386:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.062092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:05.085903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518578856952386:2048] [node 1] ICP22 created new session: [2:7571518583151919765:2048]
2025-11-11T17:19:05.129978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518583151919765:2048] [node 1] ICS09 handshake done sender: [2:7571518578856952462:2114] self: [2:7571518583151919762:0] peer: [1:7571518583610123739:0] socket: 237 qp: -1
2025-11-11T17:19:05.130040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518583151919765:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:05.141660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:05.147050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518578856952417:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(472860961ns) {Configure:104(472851399ns) {ProcessConfigured:135(8963568ns) {StartInitialHandshake:157(8961620ns) {DropHandshakes:469(1675ns) {DropIncomingHandshake:441(218ns) DropOutgoingHandshake:456(458ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:476(2205ns) {SwitchToState:178(728ns)}}}}}
2025-11-11T17:19:05.154592Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518578856952460:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.154628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518578856952460:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.167100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518583151919770:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49398
2025-11-11T17:19:05.182165Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518578856952460:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:05.185891Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518583151919770:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.186264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518583151919772:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49398
2025-11-11T17:19:05.194132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518578856952437:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(232023927ns) {Configure:104(232017990ns) {ProcessConfigured:135(83070ns) {StartInitialHandshake:157(77903ns) {DropHandshakes:469(1591ns) {DropIncomingHandshake:441(292ns) DropOutgoingHandshake:456(215ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:476(1701ns) {SwitchToState:178(405ns)}}}}}
2025-11-11T17:19:05.212542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518578856952387:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.212597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:05.212635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518578856952387:2049] [node 3] ICP22 created new session: [2:7571518583151919778:2049]
2025-11-11T17:19:05.212670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518583151919778:2049] [node 3] ICS09 handshake done sender: [2:7571518578856952460:2113] self: [2:7571518578856952459:0] peer: [3:7571518584201116153:0] socket: 233 qp: -1
2025-11-11T17:19:05.212696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518583151919778:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:05.212722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:05.227468Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518583151919772:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.277984Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518583151919768:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.278011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518583151919768:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.278863Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518583151919768:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:05.279076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518578856952417:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.279137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:05.279180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518578856952417:2070] [node 4] ICP22 created new session: [2:7571518583151919782:2070]
2025-11-11T17:19:05.279221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518583151919782:2070] [node 4] ICS09 handshake done sender: [2:7571518583151919768:2117] self: [2:7571518583151919767:0] peer: [4:7571518584265616964:0] socket: 244 qp: -1
2025-11-11T17:19:05.279248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518583151919782:2070] [node 4] ICS10 traffic start
2025-11-11T17:19:05.279279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:05.342955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518578856952386:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(215717028ns) {HandleHandshakeStatus:329(215713682ns) {DropIncomingHandshake:441(897ns) DropOutgoingHandshake:456(5737ns) SwitchToState:178(492ns) ProcessPendingSessionEvents:495(87807043ns) {ForwardSessionEventToSession:612(39810948ns) ForwardSessionEventToSession:612(13448978ns) ForwardSessionEventToSession:612(3966ns) ForwardSessionEventToSession:612(1998ns) ForwardSessionEventToSession:612(14617048ns) ForwardSessionEventToSession:612(2671ns) ForwardSessionEventToSession:612(12288762ns) ForwardSessionEventToSession:612(3387ns) ForwardSessionEventToSession:612(2371ns) ForwardSessionEventToSession:612(7605004ns)}}}
2025-11-11T17:19:05.386966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518583151919775:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:05.386999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518583151919775:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:05.399712Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518583151919775:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:05.399953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518578856952437:2077] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:05.400007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:05.400046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518578856952437:2077] [node 5] ICP22 created new session: [2:7571518583151919785:2077]
2025-11-11T17:19:05.400087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518583151919785:2077] [node 5] ICS09 handshake done sender: [2:7571518583151919775:2120] self: [2:7571518583151919774:0] peer: [5:7571518584281257148:0] socket: 261 qp: -1
2025-11-11T17:19:05.400125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518583151919785:2077] [node 5] ICS10 traffic start
2025-11-11T17:19:05.400154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:05.521857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518579970649614:2068] [node 5] ICP22 created new session: [4:7571518584265616986:2068]
2025-11-11T17:19:05.521909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518584265616986:2068] [node 5] ICS09 handshake done sender: [4:7571518584265616960:2114] self: [4:7571518584265616959:0] peer: [5:7571518584281257156:0] socket: 211 qp: -1
2025-11-11T17:19:05.521950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518584265616986:2068] [node 5] ICS10 traffic start
2025-11-11T17:19:05.522003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:05.530003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518579970649589:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:05.530084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:05.530145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518579970649589:2048] [node 1] ICP22 created new session: [4:7571518584265616988:2048]
2025-11-11T17:19:05.530191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518584265616988:2048] [node 1] ICS09 handshake done sender: [4:7571518584265616967:2117] self: [4:7571518584265616983:0] peer: [1:7571518583610123754:0] socket: 240 qp: -1
2025-11-11T17:19:05.530238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518584265616988:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:05.530313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:05.550131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518579970649614:2068] [node 5] ICP35 event processing took too much time PendingConnection:278(56126619ns) {HandleHandshakeStatus:329(46278421ns) {DropIncomingHandshake:441(16928ns) DropOutgoingHandshake:456(591ns) SwitchToState:178(644ns) ProcessPendingSessionEvents:495(27668060ns) {ForwardSessionEventToSession:612(9231366ns) ForwardSessionEventToSession:612(6251ns) ForwardSessionEventToSession:612(2667729ns) ForwardSessionEventToSession:612(4282004ns) ForwardSessionEventToSession:612(4424ns) ForwardSessionEventToSession:612(2987ns) ForwardSessionEventToSession:612(3501993ns) ForwardSessionEventToSession:612(2906ns) ForwardSessionEventToSession:612(7944062ns) ForwardSessionEventToSession:612(4054ns)}}}
2025-11-11T17:19:05.553691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:05.553768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518579986289762:2049] [node 2] ICP22 created new session: [5:7571518584281257160:2049]
2025-11-11T17:19:05.553816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518584281257160:2049] [node 2] ICS09 handshake done sender: [5:7571518584281257139:2118] self: [5:7571518584281257148:0] peer: [2:7571518583151919774:0] socket: 262 qp: -1
2025-11-11T17:19:05.553861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518584281257160:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:05.553905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:05.554154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518579986289762:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31773618ns) {HandleHandshakeStatus:329(25570985ns) {DropIncomingHandshake:441(2216ns) DropOutgoingHandshake:456(507ns) SwitchToState:178(611ns) ProcessPendingSessionEvents:495(877ns)}}
2025-11-11T17:19:05.645783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:05.646045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518583151919765:2048] [node 1] ICS07 socket disconnect 237 reason# EndOfStream
2025-11-11T17:19:05.646075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518583151919765:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.646181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:05.646219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518583151919765:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:05.655107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:05.656132Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:19:05.656809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518578856952386:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:19:05.656849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518583151919765:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:05.655327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518584201116166:2048] [node 1] ICS01 socket: 218 reason# EndOfStream
2025-11-11T17:19:05.655365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518579906148770:2048] [node 1] ICP30 unregister session Session# [3:7571518584201116166:2048] VirtualId# [3:7571518584201116143:0]
2025-11-11T17:19:05.655400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518584201116166:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.655462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:05.655497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:05.669635Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Couldn't connect to any resolved address
2025-11-11T17:19:05.669986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518578856952386:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:05.670035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518583151919765:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:05.670075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518578856952386:2048] [node 1] ICP30 unregister session Session# [2:7571518583151919765:2048] VirtualId# [2:7571518583151919762:0]
2025-11-11T17:19:05.670130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:05.670160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518578856952386:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:05.670176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26225) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:05.677749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:05.677749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:05.677987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518583169406010:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-11-11T17:19:05.678048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518584265616988:2048] [node 1] ICS01 socket: 240 reason# EndOfStream
2025-11-11T17:19:05.678027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518578874438622:2048] [node 1] ICP30 unregister session Session# [6:7571518583169406010:2048] VirtualId# [6:7571518583169406006:0]
2025-11-11T17:19:05.678063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518583169406010:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.678091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518579970649589:2048] [node 1] ICP30 unregister session Session# [4:7571518584265616988:2048] VirtualId# [4:7571518584265616983:0]
2025-11-11T17:19:05.678152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:05.678123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518584265616988:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.678198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:05.678208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:05.678245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:05.681731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:05.681980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518584281257154:2048] [node 1] ICS01 socket: 212 reason# EndOfStream
2025-11-11T17:19:05.682026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518579986289761:2048] [node 1] ICP30 unregister session Session# [5:7571518584281257154:2048] VirtualId# [5:7571518584281257147:0]
2025-11-11T17:19:05.682059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518584281257154:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.682174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:05.682219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:05.685721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518578856952386:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:05.761741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:05.761998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518584201116164:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-11T17:19:05.762048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518579906148771:2049] [node 2] ICP30 unregister session Session# [3:7571518584201116164:2049] VirtualId# [3:7571518584201116153:0]
2025-11-11T17:19:05.762083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518584201116164:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.762207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:05.762247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:05.778209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:05.777778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:05.778032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518583169406002:2049] [node 2] ICS01 socket: 225 reason# EndOfStream
2025-11-11T17:19:05.778074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518578874438623:2049] [node 2] ICP30 unregister session Session# [6:7571518583169406002:2049] VirtualId# [6:7571518578874438699:0]
2025-11-11T17:19:05.778112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518583169406002:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.778187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:05.778225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:05.778431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:05.778638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518584265616970:2049] [node 2] ICS01 socket: 252 reason# EndOfStream
2025-11-11T17:19:05.778671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518579970649590:2049] [node 2] ICP30 unregister session Session# [4:7571518584265616970:2049] VirtualId# [4:7571518584265616964:0]
2025-11-11T17:19:05.778700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518584265616970:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.778768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:05.778805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:05.778409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518584281257160:2049] [node 2] ICS01 socket: 262 reason# EndOfStream
2025-11-11T17:19:05.778451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518579986289762:2049] [node 2] ICP30 unregister session Session# [5:7571518584281257160:2049] VirtualId# [5:7571518584281257148:0]
2025-11-11T17:19:05.778480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518584281257160:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.778551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:05.778587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:05.913754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:05.913825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:05.913990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518584281257140:2050] [node 3] ICS01 socket: 220 reason# EndOfStream
2025-11-11T17:19:05.914024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518579986289763:2050] [node 3] ICP30 unregister session Session# [5:7571518584281257140:2050] VirtualId# [5:7571518584281257135:0]
2025-11-11T17:19:05.914049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518584281257140:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.914103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:05.914140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:05.913754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:05.913968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518584265616977:2050] [node 3] ICS07 socket disconnect 213 reason# EndOfStream
2025-11-11T17:19:05.913993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518584265616977:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.914077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:05.914124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518584265616977:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:05.915474Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10960) Couldn't connect to any resolved address
2025-11-11T17:19:05.915067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518583169406013:2050] [node 3] ICS01 socket: 247 reason# EndOfStream
2025-11-11T17:19:05.915108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518578874438624:2050] [node 3] ICP30 unregister session Session# [6:7571518583169406013:2050] VirtualId# [6:7571518583169406007:0]
2025-11-11T17:19:05.915143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518583169406013:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.915201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:05.915240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:05.985739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:05.986058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518583169406020:2068] [node 4] ICS01 socket: 273 reason# EndOfStream
2025-11-11T17:19:05.986119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518578874438696:2068] [node 4] ICP30 unregister session Session# [6:7571518583169406020:2068] VirtualId# [6:7571518583169406018:0]
2025-11-11T17:19:05.986176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518583169406020:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.986286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:05.986330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:05.986349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:05.986504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518584281257158:2070] [node 4] ICS01 socket: 227 reason# EndOfStream
2025-11-11T17:19:05.986542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518579986289790:2070] [node 4] ICP30 unregister session Session# [5:7571518584281257158:2070] VirtualId# [5:7571518584281257156:0]
2025-11-11T17:19:05.986564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518584281257158:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:05.986619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:05.986653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:06.063540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:06.063774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518583169406014:2059] [node 5] ICS01 socket: 230 reason# EndOfStream
2025-11-11T17:19:06.063823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518578874438681:2059] [node 5] ICP30 unregister session Session# [6:7571518583169406014:2059] VirtualId# [6:7571518583169406005:0]
2025-11-11T17:19:06.063858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518583169406014:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:06.063945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:06.063989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:06.269985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518587823820140:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:06.270033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518587823820141:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:06.321701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518590013704980:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:06.321754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518590013704981:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:06.365809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518589392423674:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:06.365860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518589392423675:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:06.449826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518589217954294:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:06.449888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518589217954295:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:06.449915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518589217954296:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:06.504409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518590182825326:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:06.504465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518590182825327:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:06.504491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518590182825328:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:06.790519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518590182825340:2060] [node 4] ICP01 ready to work
2025-11-11T17:19:06.862299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518590013705037:2089] [node 6] ICP01 ready to work
2025-11-11T17:19:06.705867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518590039048082:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:06.858634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518590182825350:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:06.885707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518589392423735:2072] [node 5] ICP01 ready to work
2025-11-11T17:19:06.901912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518589217954334:2060] [node 5] ICP01 ready to work
2025-11-11T17:19:06.907574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518590013705039:2091] [node 5] ICP01 ready to work
2025-11-11T17:19:06.937012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518590013705045:2092] [node 4] ICP01 ready to work
2025-11-11T17:19:06.705933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518590039048083:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:06.929243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518587823820140:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(161987996ns) {Configure:104(161978774ns) {ProcessConfigured:135(27314833ns) {StartInitialHandshake:157(19412673ns) {DropHandshakes:469(5305ns) {DropIncomingHandshake:441(938ns) DropOutgoingHandshake:456(640ns)} GenerateSessionVirtualId:988(2422ns) PrepareNewSessionHandshake:476(5192ns) {SwitchToState:178(2232ns)}}}}}
2025-11-11T17:19:06.705960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518590039048084:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:06.937666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518589392423738:2073] [node 4] ICP01 ready to work
2025-11-11T17:19:06.957692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518589392423739:2074] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.033550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518594308672342:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33056
2025-11-11T17:19:06.929671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518587823820205:2094] [node 6] ICP01 ready to work
2025-11-11T17:19:06.936764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518590039048099:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:07.075293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518590039048084:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(138290809ns) {Configure:104(138287182ns) {ProcessConfigured:135(46420ns) {StartInitialHandshake:157(43960ns) {DropHandshakes:469(2031ns) {DropIncomingHandshake:441(541ns) DropOutgoingHandshake:456(486ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:476(1709ns) {SwitchToState:178(960ns)}}}}}
2025-11-11T17:19:06.929958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518587823820206:2096] [node 5] ICP01 ready to work
2025-11-11T17:19:06.930247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518587823820207:2097] [node 4] ICP01 ready to work
2025-11-11T17:19:07.102084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518587823820205:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(169476559ns) {Configure:104(169471842ns) {ProcessConfigured:135(53174ns) {StartInitialHandshake:157(51590ns) {DropHandshakes:469(1989ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(328ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(1419ns) {SwitchToState:178(489ns)}}}}}
2025-11-11T17:19:07.090919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518594334015448:2072] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.089883Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672342:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.129883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518587823820207:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25080783ns) {Configure:104(25077818ns) {ProcessConfigured:135(8032836ns) {StartInitialHandshake:157(8030099ns) {DropHandshakes:469(1974ns) {DropIncomingHandshake:441(255ns) DropOutgoingHandshake:456(396ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:476(5634ns) {SwitchToState:178(1495ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.093689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518589217954334:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(92909870ns) {Configure:104(92902705ns) {ProcessConfigured:135(84514ns) {StartInitialHandshake:157(78066ns) {DropHandshakes:469(3876ns) {DropIncomingHandshake:441(627ns) DropOutgoingHandshake:456(695ns)} GenerateSessionVirtualId:988(887ns) PrepareNewSessionHandshake:476(6373ns) {SwitchToState:178(639ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.138651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518593687391036:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32365
2025-11-11T17:19:07.139503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518593687391047:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32365
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.117928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792699:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.094073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518593512921657:2097] [node 6] ICP01 ready to work
2025-11-11T17:19:07.131955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518589217954295:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(35165751ns) {Configure:104(35162666ns) {ProcessConfigured:135(49585ns) {StartInitialHandshake:157(47935ns) {DropHandshakes:469(2797ns) {DropIncomingHandshake:441(516ns) DropOutgoingHandshake:456(549ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:476(1848ns) {SwitchToState:178(1009ns)}}}}}
2025-11-11T17:19:07.133891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518589217954358:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
2025-11-11T17:19:07.134346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518593512921664:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
2025-11-11T17:19:07.145870Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518589217954358:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.147471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518593512921670:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
2025-11-11T17:19:07.131255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792701:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.142107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792702:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.142166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792703:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.177300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015449:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
2025-11-11T17:19:07.154533Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391036:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.185634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015450:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
2025-11-11T17:19:07.185823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015451:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.189553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518594308672357:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33056
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.207750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518592118787515:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57846
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.225779Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672346:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.225827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518594308672346:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.226731Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672357:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.227308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518590013704980:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.227418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518594308672358:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33056
2025-11-11T17:19:07.209901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518594477792703:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.218176Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787515:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.237838Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518594308672346:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:07.239308Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672348:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.239336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518594308672348:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.240047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518590013705039:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.240105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:07.240153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518590013705039:2091] [node 5] ICP22 created new session: [2:7571518594308672362:2091]
2025-11-11T17:19:07.240201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518594308672362:2091] [node 5] ICS09 handshake done sender: [2:7571518594308672346:2112] self: [2:7571518594308672345:0] peer: [5:7571518594477792705:0] socket: 223 qp: -1
2025-11-11T17:19:07.240236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518594308672362:2091] [node 5] ICS10 traffic start
2025-11-11T17:19:07.240270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:07.213083Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391038:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.227965Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518593512921664:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.228089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518589217954295:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.211023Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518594334015451:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.213118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518593687391038:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.254929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518593687391052:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32365
2025-11-11T17:19:07.255268Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518593687391038:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:07.255673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518589392423735:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.255780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:07.255826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518589392423735:2072] [node 5] ICP22 created new session: [3:7571518593687391054:2072]
2025-11-11T17:19:07.255879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518593687391054:2072] [node 5] ICS09 handshake done sender: [3:7571518593687391038:2111] self: [3:7571518593687391037:0] peer: [5:7571518594477792704:0] socket: 236 qp: -1
2025-11-11T17:19:07.255912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518593687391054:2072] [node 5] ICS10 traffic start
2025-11-11T17:19:07.255951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:07.228322Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518587823820204:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.228345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518587823820204:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.229286Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518587823820204:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:07.229626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518587823820140:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.229670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:07.237681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518587823820140:2048] [node 2] ICP22 created new session: [1:7571518592118787518:2048]
2025-11-11T17:19:07.249840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518592118787518:2048] [node 2] ICS09 handshake done sender: [1:7571518587823820204:2110] self: [1:7571518587823820202:0] peer: [2:7571518594308672360:0] socket: 226 qp: -1
2025-11-11T17:19:07.258016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518592118787518:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.210027Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518594477792699:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.258057Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518594334015449:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.262665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518590039048084:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.266151Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391047:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.210187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518590182825328:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.210245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518590182825327:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.238171Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518594477792702:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.238650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792706:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.238702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792707:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.241629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518590182825340:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.262310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518594308672361:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33056
2025-11-11T17:19:07.269843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:07.283053Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518594308672357:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.288003Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:07.289877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391042:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.289909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518593687391042:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.290744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518593687391058:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32365
2025-11-11T17:19:07.291157Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518593687391042:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:07.291401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518589392423739:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.291466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:07.291514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518589392423739:2074] [node 6] ICP22 created new session: [3:7571518593687391060:2074]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.292376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518590039048083:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.293011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015461:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
2025-11-11T17:19:07.291555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518593687391060:2074] [node 6] ICS09 handshake done sender: [3:7571518593687391042:2113] self: [3:7571518593687391041:0] peer: [6:7571518594334015459:0] socket: 238 qp: -1
2025-11-11T17:19:07.281926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518593512921670:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.306552Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:07.307637Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518594477792699:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:07.307978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518590182825327:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518594477792699:2114] held: no
2025-11-11T17:19:07.308066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518590182825327:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.308105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:07.308147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518590182825327:2049] [node 2] ICP22 created new session: [5:7571518594477792710:2049]
2025-11-11T17:19:07.308186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518594477792710:2049] [node 2] ICS09 handshake done sender: [5:7571518594477792699:2114] self: [5:7571518594477792705:0] peer: [2:7571518594308672345:0] socket: 225 qp: -1
2025-11-11T17:19:07.308216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518594477792710:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:07.308248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:07.308996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518590182825328:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518594477792703:2118] held: no
2025-11-11T17:19:07.291585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518593687391060:2074] [node 6] ICS10 traffic start
2025-11-11T17:19:07.291618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.282057Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518593512921656:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.282077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518593512921656:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.318841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518589217954296:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.318932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518593512921673:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.326922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518587823820140:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(97096497ns) {HandleHandshakeStatus:329(97093767ns) {DropIncomingHandshake:441(592ns) DropOutgoingHandshake:456(1008ns) SwitchToState:178(384ns) ProcessPendingSessionEvents:495(7143563ns) {ForwardSessionEventToSession:612(7089965ns) ForwardSessionEventToSession:612(8408ns) ForwardSessionEventToSession:612(5682ns) ForwardSessionEventToSession:612(2471ns) ForwardSessionEventToSession:612(4621ns) ForwardSessionEventToSession:612(2191ns) ForwardSessionEventToSession:612(2979ns) ForwardSessionEventToSession:612(5625ns) ForwardSessionEventToSession:612(2632ns) ForwardSessionEventToSession:612(4690ns)}}}
2025-11-11T17:19:07.329748Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518593512921656:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:07.321720Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391052:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.319392Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518594334015451:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:07.333994Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518594334015450:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.334033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518590039048084:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.334113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:07.334157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518590039048084:2050] [node 3] ICP22 created new session: [6:7571518594334015462:2050]
2025-11-11T17:19:07.334191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518594334015462:2050] [node 3] ICS09 handshake done sender: [6:7571518594334015451:2115] self: [6:7571518594334015459:0] peer: [3:7571518593687391041:0] socket: 244 qp: -1
2025-11-11T17:19:07.334222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518594334015462:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:07.334256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.326275Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518594477792706:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.326497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792709:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.321910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391040:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.321931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518593687391040:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.322599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518589392423739:2074] [node 6] ICP18 handshake (actor [3:7571518593687391052:2117]) is too old
2025-11-11T17:19:07.342219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518594308672365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33056
2025-11-11T17:19:07.342463Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518594308672348:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.358830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518592118787520:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57846
2025-11-11T17:19:07.358940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518592118787521:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57846
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.350122Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518594477792701:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.350578Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518594477792703:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:07.362305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792712:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.369880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518590039048082:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.364146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518590013704980:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.361851Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518593687391040:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:07.377148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518589217954334:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.377245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:07.377290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518589217954334:2060] [node 5] ICP22 created new session: [4:7571518593512921677:2060]
2025-11-11T17:19:07.377340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518593512921677:2060] [node 5] ICS09 handshake done sender: [4:7571518593512921656:2111] self: [4:7571518593512921655:0] peer: [5:7571518594477792708:0] socket: 231 qp: -1
2025-11-11T17:19:07.377374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518593512921677:2060] [node 5] ICS10 traffic start
2025-11-11T17:19:07.377435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:07.382661Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518593512921664:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.393879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518590182825328:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.393937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:07.393981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518590182825328:2050] [node 3] ICP22 created new session: [5:7571518594477792716:2050]
2025-11-11T17:19:07.394028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518594477792716:2050] [node 3] ICS09 handshake done sender: [5:7571518594477792703:2118] self: [5:7571518594477792704:0] peer: [3:7571518593687391037:0] socket: 239 qp: -1
2025-11-11T17:19:07.394061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518594477792716:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:07.394093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:07.394615Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.364217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:07.402672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015464:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
2025-11-11T17:19:07.402765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015466:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.386589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518589392423738:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.395779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518593512921676:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
2025-11-11T17:19:07.401743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787520:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.425671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787505:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.425733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518592118787505:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.364259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518590013704980:2048] [node 1] ICP22 created new session: [2:7571518594308672367:2048]
2025-11-11T17:19:07.364301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518594308672367:2048] [node 1] ICS09 handshake done sender: [2:7571518594308672357:2116] self: [2:7571518594308672360:0] peer: [1:7571518587823820202:0] socket: 241 qp: -1
2025-11-11T17:19:07.364334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518594308672367:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:07.364368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:07.365000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672344:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.365023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518594308672344:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.365667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518590013705045:2092] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.365703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:07.365741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518590013705045:2092] [node 4] ICP22 created new session: [2:7571518594308672369:2092]
2025-11-11T17:19:07.369661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518594308672369:2092] [node 4] ICS09 handshake done sender: [2:7571518594308672348:2113] self: [2:7571518594308672347:0] peer: [4:7571518593512921672:0] socket: 224 qp: -1
2025-11-11T17:19:07.369705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518594308672369:2092] [node 4] ICS10 traffic start
2025-11-11T17:19:07.369743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:07.397765Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518594308672344:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:07.434347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518590182825326:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.434548Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518594477792702:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.437879Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672358:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.437970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672365:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.439237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518592118787524:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57846
2025-11-11T17:19:07.386673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:07.386731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518589392423738:2073] [node 4] ICP22 created new session: [3:7571518593687391065:2073]
2025-11-11T17:19:07.386799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518593687391065:2073] [node 4] ICS09 handshake done sender: [3:7571518593687391040:2112] self: [3:7571518593687391039:0] peer: [4:7571518593512921675:0] socket: 237 qp: -1
2025-11-11T17:19:07.386835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518593687391065:2073] [node 4] ICS10 traffic start
2025-11-11T17:19:07.386893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:07.410404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518593687391064:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32365
2025-11-11T17:19:07.442275Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391058:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.442394Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518593687391064:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.442544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518589392423675:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.442598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518589392423674:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.454359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518593687391069:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32365
2025-11-11T17:19:07.435027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518590182825340:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518594477792702:2117] held: no
2025-11-11T17:19:07.442084Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518594334015449:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:07.445881Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518594308672350:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.465733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518589217954295:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.469170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.473974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518593687391070:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32365
2025-11-11T17:19:07.473701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518589217954295:2049] [node 2] ICP22 created new session: [4:7571518593512921680:2049]
2025-11-11T17:19:07.473774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518593512921680:2049] [node 2] ICS09 handshake done sender: [4:7571518593512921664:2115] self: [4:7571518593512921672:0] peer: [2:7571518594308672347:0] socket: 230 qp: -1
2025-11-11T17:19:07.435127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518590182825340:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.448128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518592118787505:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:07.467405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518590039048083:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.467513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:07.467559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518590039048083:2049] [node 2] ICP22 created new session: [6:7571518594334015469:2049]
2025-11-11T17:19:07.467605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518594334015469:2049] [node 2] ICS09 handshake done sender: [6:7571518594334015449:2113] self: [6:7571518594334015460:0] peer: [2:7571518594308672343:0] socket: 242 qp: -1
2025-11-11T17:19:07.467641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518594334015469:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:07.467677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:07.468372Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.445916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518594308672350:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.481749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518593512921680:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:07.470151Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787509:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.470176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518592118787509:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.471109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518587823820205:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.471145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:07.471181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518587823820205:2094] [node 6] ICP22 created new session: [1:7571518592118787526:2094]
2025-11-11T17:19:07.471235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518592118787526:2094] [node 6] ICS09 handshake done sender: [1:7571518592118787505:2111] self: [1:7571518592118787504:0] peer: [6:7571518594334015465:0] socket: 227 qp: -1
2025-11-11T17:19:07.471265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518592118787526:2094] [node 6] ICS10 traffic start
2025-11-11T17:19:07.471308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:07.485844Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518592118787509:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:07.435205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:07.446696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518590013705037:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.523049Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787507:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.523088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518592118787507:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.435251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518590182825340:2060] [node 4] ICP22 created new session: [5:7571518594477792719:2060]
2025-11-11T17:19:07.481856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:07.482359Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518589217954295:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16433437ns) {HandleHandshakeStatus:329(16428914ns) {DropIncomingHandshake:441(997ns) DropOutgoingHandshake:456(6514ns) SwitchToState:178(658ns) ProcessPendingSessionEvents:495(120739ns) {ForwardSessionEventToSession:612(62818ns) ForwardSessionEventToSession:612(7941ns) ForwardSessionEventToSession:612(3970ns) ForwardSessionEventToSession:612(12231ns) ForwardSessionEventToSession:612(5875ns) ForwardSessionEventToSession:612(2388ns) ForwardSessionEventToSession:612(2886ns) ForwardSessionEventToSession:612(5152ns) ForwardSessionEventToSession:612(2952ns) ForwardSessionEventToSession:612(5608ns)}}}
2025-11-11T17:19:07.482550Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:07.491413Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518593512921670:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:07.491710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518589217954296:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518593512921670:2117] held: no
2025-11-11T17:19:07.491814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518589217954296:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.491883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:07.523896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518587823820141:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.446734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:07.561770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518590039048083:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:07.491925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518589217954296:2050] [node 3] ICP22 created new session: [4:7571518593512921683:2050]
2025-11-11T17:19:07.530172Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:07.531340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518589392423674:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518593687391064:2119] held: no
2025-11-11T17:19:07.531448Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518593687391058:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:07.531626Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518593687391064:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:07.531816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518589392423675:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.531859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:07.531896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518589392423675:2049] [node 2] ICP22 created new session: [3:7571518593687391071:2049]
2025-11-11T17:19:07.531944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518593687391071:2049] [node 2] ICS09 handshake done sender: [3:7571518593687391058:2118] self: [3:7571518593687391067:0] peer: [2:7571518594308672349:0] socket: 259 qp: -1
2025-11-11T17:19:07.531978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518593687391071:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:07.532029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:07.532413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518589392423674:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.532433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:07.532462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518589392423674:2048] [node 1] ICP22 created new session: [3:7571518593687391073:2048]
2025-11-11T17:19:07.532493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518593687391073:2048] [node 1] ICS09 handshake done sender: [3:7571518593687391064:2119] self: [3:7571518593687391068:0] peer: [1:7571518592118787508:0] socket: 219 qp: -1
2025-11-11T17:19:07.532516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518593687391073:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:07.532536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:07.523953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:07.491967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518593512921683:2050] [node 3] ICS09 handshake done sender: [4:7571518593512921670:2117] self: [4:7571518593512921675:0] peer: [3:7571518593687391039:0] socket: 240 qp: -1
2025-11-11T17:19:07.492001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518593512921683:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:07.492041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:07.492794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518593512921682:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
2025-11-11T17:19:07.569740Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518593512921682:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.569952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518589217954294:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.523998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518587823820141:2049] [node 3] ICP22 created new session: [1:7571518592118787530:2049]
2025-11-11T17:19:07.524037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518592118787530:2049] [node 3] ICS09 handshake done sender: [1:7571518592118787509:2113] self: [1:7571518592118787508:0] peer: [3:7571518593687391068:0] socket: 268 qp: -1
2025-11-11T17:19:07.524076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518592118787530:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:07.524109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:07.544155Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787521:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.550942Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518592118787507:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:07.553546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518587823820206:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.561864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:07.561945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518587823820206:2096] [node 5] ICP22 created new session: [1:7571518592118787534:2096]
2025-11-11T17:19:07.562001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518592118787534:2096] [node 5] ICS09 handshake done sender: [1:7571518592118787507:2112] self: [1:7571518592118787506:0] peer: [5:7571518594477792718:0] socket: 228 qp: -1
2025-11-11T17:19:07.562039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518592118787534:2096] [node 5] ICS10 traffic start
2025-11-11T17:19:07.562083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:07.571216Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787511:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.571240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518592118787511:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.446781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518590013705037:2089] [node 6] ICP22 created new session: [2:7571518594308672372:2089]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.576263Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518594334015464:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.576363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015468:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
2025-11-11T17:19:07.446827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518594308672372:2089] [node 6] ICS09 handshake done sender: [2:7571518594308672344:2111] self: [2:7571518594308672343:0] peer: [6:7571518594334015460:0] socket: 222 qp: -1
2025-11-11T17:19:07.446863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518594308672372:2089] [node 6] ICS10 traffic start
2025-11-11T17:19:07.446901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:07.447679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518590013705037:2089] [node 6] ICP18 handshake (actor [2:7571518594308672358:2117]) is too old
2025-11-11T17:19:07.477945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518594308672350:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:07.526177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518590013704981:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.526227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:07.526267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518590013704981:2049] [node 3] ICP22 created new session: [2:7571518594308672376:2049]
2025-11-11T17:19:07.526306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518594308672376:2049] [node 3] ICS09 handshake done sender: [2:7571518594308672350:2114] self: [2:7571518594308672349:0] peer: [3:7571518593687391067:0] socket: 258 qp: -1
2025-11-11T17:19:07.526338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518594308672376:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:07.526371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:07.435294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518594477792719:2060] [node 4] ICS09 handshake done sender: [5:7571518594477792702:2117] self: [5:7571518594477792708:0] peer: [4:7571518593512921655:0] socket: 234 qp: -1
2025-11-11T17:19:07.435335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518594477792719:2060] [node 4] ICS10 traffic start
2025-11-11T17:19:07.435371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:07.544465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518594477792721:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-11T17:19:07.551535Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518594477792701:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:07.551938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518590182825326:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.551994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:07.552041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518590182825326:2048] [node 1] ICP22 created new session: [5:7571518594477792722:2048]
2025-11-11T17:19:07.552080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518594477792722:2048] [node 1] ICS09 handshake done sender: [5:7571518594477792701:2116] self: [5:7571518594477792718:0] peer: [1:7571518592118787506:0] socket: 229 qp: -1
2025-11-11T17:19:07.552115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518594477792722:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:07.552152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.578035Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518592118787524:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.578322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518587823820205:2094] [node 6] ICP18 handshake (actor [1:7571518592118787524:2118]) is too old
2025-11-11T17:19:07.579932Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518594334015450:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.580618Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518592118787511:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:07.578865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518593512921687:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
2025-11-11T17:19:07.580282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518593512921688:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49236
2025-11-11T17:19:07.581104Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518593512921682:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:07.581302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518589217954294:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.581346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:07.581394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518589217954294:2048] [node 1] ICP22 created new session: [4:7571518593512921689:2048]
2025-11-11T17:19:07.581435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518593512921689:2048] [node 1] ICS09 handshake done sender: [4:7571518593512921682:2120] self: [4:7571518593512921686:0] peer: [1:7571518592118787510:0] socket: 233 qp: -1
2025-11-11T17:19:07.581464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518593512921689:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:07.581497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.583100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015471:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
2025-11-11T17:19:07.583243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518590039048099:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.583343Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:07.584469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518587823820207:2097] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.584525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:07.584569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518587823820207:2097] [node 4] ICP22 created new session: [1:7571518592118787537:2097]
2025-11-11T17:19:07.584616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518592118787537:2097] [node 4] ICS09 handshake done sender: [1:7571518592118787511:2114] self: [1:7571518592118787510:0] peer: [4:7571518593512921686:0] socket: 277 qp: -1
2025-11-11T17:19:07.584648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518592118787537:2097] [node 4] ICS10 traffic start
2025-11-11T17:19:07.584705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:07.592003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518590039048082:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.592068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:07.592109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518590039048082:2048] [node 1] ICP22 created new session: [6:7571518594334015474:2048]
2025-11-11T17:19:07.592151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518594334015474:2048] [node 1] ICS09 handshake done sender: [6:7571518594334015450:2114] self: [6:7571518594334015465:0] peer: [1:7571518592118787504:0] socket: 243 qp: -1
2025-11-11T17:19:07.592189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518594334015474:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:07.592223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:07.592428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518590039048082:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:07.592934Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518594334015471:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.594309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518594334015448:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:07.594936Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518590182825354:2067] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.594974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518590182825354:2067] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.595970Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518590182825354:2067] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:07.596301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518590182825350:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.596358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:07.596399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518590182825350:2065] [node 6] ICP22 created new session: [5:7571518594477792725:2065]
2025-11-11T17:19:07.596436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518594477792725:2065] [node 6] ICS09 handshake done sender: [5:7571518590182825354:2067] self: [5:7571518590182825353:0] peer: [6:7571518594334015473:0] socket: 247 qp: -1
2025-11-11T17:19:07.596469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518594477792725:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:07.596495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:07.597788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518593512921666:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.597822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518593512921666:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:07.599258Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518593512921666:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:07.599640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518593512921657:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:07.599730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:07.599765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518593512921657:2097] [node 6] ICP22 created new session: [4:7571518593512921692:2097]
2025-11-11T17:19:07.599802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518593512921692:2097] [node 6] ICS09 handshake done sender: [4:7571518593512921666:2116] self: [4:7571518593512921665:0] peer: [6:7571518594334015476:0] socket: 248 qp: -1
2025-11-11T17:19:07.599829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518593512921692:2097] [node 6] ICS10 traffic start
2025-11-11T17:19:07.599858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:07.600779Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:07.601641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518593512921688:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:07.601737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518593512921657:2097] [node 6] ICP18 handshake (actor [4:7571518593512921688:2122]) is too old
2025-11-11T17:19:07.602607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518590039048099:2062] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518594334015464:2119] held: no
2025-11-11T17:19:07.602725Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.605997Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518594334015448:2072] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518594334015471:2122] held: no
2025-11-11T17:19:07.606138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015477:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:07.606564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518594334015478:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3652
2025-11-11T17:19:07.607012Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518594334015464:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:07.607285Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518594334015471:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:07.615098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518590039048099:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.615162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:07.615216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518590039048099:2062] [node 5] ICP22 created new session: [6:7571518594334015479:2062]
2025-11-11T17:19:07.615257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518594334015479:2062] [node 5] ICS09 handshake done sender: [6:7571518594334015464:2119] self: [6:7571518594334015473:0] peer: [5:7571518590182825353:0] socket: 269 qp: -1
2025-11-11T17:19:07.615298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518594334015479:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:07.615333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:07.615747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518594334015448:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:07.615818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:07.615865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518594334015448:2072] [node 4] ICP22 created new session: [6:7571518594334015481:2072]
2025-11-11T17:19:07.615916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518594334015481:2072] [node 4] ICS09 handshake done sender: [6:7571518594334015471:2122] self: [6:7571518594334015476:0] peer: [4:7571518593512921665:0] socket: 246 qp: -1
2025-11-11T17:19:07.615950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518594334015481:2072] [node 4] ICS10 traffic start
2025-11-11T17:19:07.615979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:07.681753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:07.681753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:07.682010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518593687391073:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-11T17:19:07.682051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518589392423674:2048] [node 1] ICP30 unregister session Session# [3:7571518593687391073:2048] VirtualId# [3:7571518593687391068:0]
2025-11-11T17:19:07.682084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518593687391073:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.682163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:07.682202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:07.682635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518594308672367:2048] [node 1] ICS01 socket: 241 reason# EndOfStream
2025-11-11T17:19:07.682674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518590013704980:2048] [node 1] ICP30 unregister session Session# [2:7571518594308672367:2048] VirtualId# [2:7571518594308672360:0]
2025-11-11T17:19:07.682694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518594308672367:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.682740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:07.682774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:07.697854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:07.698089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518594334015474:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-11T17:19:07.698127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518590039048082:2048] [node 1] ICP30 unregister session Session# [6:7571518594334015474:2048] VirtualId# [6:7571518594334015465:0]
2025-11-11T17:19:07.698161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518594334015474:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.698236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:07.698278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:07.698353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:07.698511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518594477792722:2048] [node 1] ICS01 socket: 229 reason# EndOfStream
2025-11-11T17:19:07.698537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518590182825326:2048] [node 1] ICP30 unregister session Session# [5:7571518594477792722:2048] VirtualId# [5:7571518594477792718:0]
2025-11-11T17:19:07.698556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518594477792722:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.698598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:07.698630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:07.698696Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:07.698838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518593512921689:2048] [node 1] ICS01 socket: 233 reason# EndOfStream
2025-11-11T17:19:07.698864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518589217954294:2048] [node 1] ICP30 unregister session Session# [4:7571518593512921689:2048] VirtualId# [4:7571518593512921686:0]
2025-11-11T17:19:07.698886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518593512921689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.698922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:07.698951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:07.799534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:07.799774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518593687391071:2049] [node 2] ICS01 socket: 259 reason# EndOfStream
2025-11-11T17:19:07.799808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518589392423675:2049] [node 2] ICP30 unregister session Session# [3:7571518593687391071:2049] VirtualId# [3:7571518593687391067:0]
2025-11-11T17:19:07.799838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518593687391071:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.799911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:07.799945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:07.813770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:07.813974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518594334015469:2049] [node 2] ICS01 socket: 242 reason# EndOfStream
2025-11-11T17:19:07.814009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518590039048083:2049] [node 2] ICP30 unregister session Session# [6:7571518594334015469:2049] VirtualId# [6:7571518594334015460:0]
2025-11-11T17:19:07.814054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518594334015469:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.814143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:07.814184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:07.814275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:07.814451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518594477792710:2049] [node 2] ICS01 socket: 225 reason# EndOfStream
2025-11-11T17:19:07.814485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518590182825327:2049] [node 2] ICP30 unregister session Session# [5:7571518594477792710:2049] VirtualId# [5:7571518594477792705:0]
2025-11-11T17:19:07.814505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518594477792710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.814562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:07.814593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:07.814673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:07.814823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518593512921680:2049] [node 2] ICS01 socket: 230 reason# EndOfStream
2025-11-11T17:19:07.814849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518589217954295:2049] [node 2] ICP30 unregister session Session# [4:7571518593512921680:2049] VirtualId# [4:7571518593512921672:0]
2025-11-11T17:19:07.814868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518593512921680:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.814910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:07.814937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:07.860863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:07.860863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:07.860884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:07.861081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518593512921683:2050] [node 3] ICS01 socket: 240 reason# EndOfStream
2025-11-11T17:19:07.861079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518594477792716:2050] [node 3] ICS01 socket: 239 reason# EndOfStream
2025-11-11T17:19:07.861080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518594334015462:2050] [node 3] ICS01 socket: 244 reason# EndOfStream
2025-11-11T17:19:07.861113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518589217954296:2050] [node 3] ICP30 unregister session Session# [4:7571518593512921683:2050] VirtualId# [4:7571518593512921675:0]
2025-11-11T17:19:07.861113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518590182825328:2050] [node 3] ICP30 unregister session Session# [5:7571518594477792716:2050] VirtualId# [5:7571518594477792704:0]
2025-11-11T17:19:07.861142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518593512921683:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.861111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518590039048084:2050] [node 3] ICP30 unregister session Session# [6:7571518594334015462:2050] VirtualId# [6:7571518594334015459:0]
2025-11-11T17:19:07.861140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518594477792716:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.861141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518594334015462:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:07.861216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:07.861212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:07.861216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:07.861254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:07.861251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:07.861253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:08.003181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:08.003424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518594477792719:2060] [node 4] ICS01 socket: 234 reason# EndOfStream
2025-11-11T17:19:08.003462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518590182825340:2060] [node 4] ICP30 unregister session Session# [5:7571518594477792719:2060] VirtualId# [5:7571518594477792708:0]
2025-11-11T17:19:08.003497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518594477792719:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:08.003598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:08.003641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:08.009719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:08.009967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518594334015481:2072] [node 4] ICS01 socket: 246 reason# EndOfStream
2025-11-11T17:19:08.010007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518594334015448:2072] [node 4] ICP30 unregister session Session# [6:7571518594334015481:2072] VirtualId# [6:7571518594334015476:0]
2025-11-11T17:19:08.010043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518594334015481:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:08.010185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:08.010234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:08.051277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:08.051524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518594334015479:2062] [node 5] ICS01 socket: 269 reason# EndOfStream
2025-11-11T17:19:08.051577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518590039048099:2062] [node 5] ICP30 unregister session Session# [6:7571518594334015479:2062] VirtualId# [6:7571518594334015473:0]
2025-11-11T17:19:08.051619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518594334015479:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:08.051732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:08.051778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:08.132228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518596933288319:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:08.132270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518596933288320:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:08.161957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518597630209382:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:08.162001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518597630209383:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:08.192371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518597013765800:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:08.192415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518597013765801:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:08.248658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518597474255392:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:08.248700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518597474255393:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:08.248724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518597474255394:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:08.309946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518597734393535:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:08.309989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518597734393536:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:08.310018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518597734393537:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:08.462257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518597013765819:2063] [node 4] ICP01 ready to work
2025-11-11T17:19:08.462739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518597013765820:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:08.467567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518597474255411:2062] [node 6] ICP01 ready to work
2025-11-11T17:19:08.463724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518596933288339:2061] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.467525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518597474255409:2061] [node 5] ICP01 ready to work
2025-11-11T17:19:08.462604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518597630209402:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:08.463048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518597630209403:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:08.472509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518597630209404:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:08.461873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518597734393549:2060] [node 6] ICP01 ready to work
2025-11-11T17:19:08.526764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518597630209402:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(63369890ns) {Configure:104(63362224ns) {ProcessConfigured:135(75386ns) {StartInitialHandshake:157(72741ns) {DropHandshakes:469(3061ns) {DropIncomingHandshake:441(698ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(805ns) PrepareNewSessionHandshake:476(2227ns) {SwitchToState:178(610ns)}}}}}
2025-11-11T17:19:08.482582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518597013765819:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19582784ns) {Configure:104(19576012ns) {ProcessConfigured:135(63152ns) {StartInitialHandshake:157(60707ns) {DropHandshakes:469(3072ns) {DropIncomingHandshake:441(638ns) DropOutgoingHandshake:456(547ns)} GenerateSessionVirtualId:988(808ns) PrepareNewSessionHandshake:476(1782ns) {SwitchToState:178(570ns)}}}}}
2025-11-11T17:19:08.499138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518597734393549:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37001590ns) {Configure:104(36998042ns) {ProcessConfigured:135(64954ns) {StartInitialHandshake:157(62028ns) {DropHandshakes:469(3204ns) {DropIncomingHandshake:441(718ns) DropOutgoingHandshake:456(600ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:476(1669ns) {SwitchToState:178(636ns)}}}}}
2025-11-11T17:19:08.484309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518597013765826:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
2025-11-11T17:19:08.490606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518597013765801:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(48255171ns) {Configure:104(48249851ns) {ProcessConfigured:135(14794830ns) {StartInitialHandshake:157(14792013ns) {DropHandshakes:469(3528ns) {DropIncomingHandshake:441(648ns) DropOutgoingHandshake:456(733ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:476(2096ns) {SwitchToState:178(943ns)}}}}}
2025-11-11T17:19:08.529771Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address
2025-11-11T17:19:08.529941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518596933288319:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21482140ns) {Configure:104(21476648ns) {ProcessConfigured:135(20099492ns) {StartInitialHandshake:157(20097067ns) {DropHandshakes:469(4749ns) {DropIncomingHandshake:441(760ns) DropOutgoingHandshake:456(443ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:476(2396ns) {SwitchToState:178(632ns)}}}}}
2025-11-11T17:19:08.510789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518597734393536:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(48886481ns) {Configure:104(48880617ns) {ProcessConfigured:135(121543ns) {StartInitialHandshake:157(118651ns) {DropHandshakes:469(3309ns) {DropIncomingHandshake:441(734ns) DropOutgoingHandshake:456(644ns)} GenerateSessionVirtualId:988(951ns) PrepareNewSessionHandshake:476(2370ns) {SwitchToState:178(897ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.549951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518597013765820:2065] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:08.567273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518597474255394:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(89398343ns) {Configure:104(89394694ns) {ProcessConfigured:135(62005ns) {StartInitialHandshake:157(59454ns) {DropHandshakes:469(3260ns) {DropIncomingHandshake:441(484ns) DropOutgoingHandshake:456(562ns)} GenerateSessionVirtualId:988(673ns) PrepareNewSessionHandshake:476(1898ns) {SwitchToState:178(634ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.550025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518597013765820:2065] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.579280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518597474255418:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:08.579341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518597474255419:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:08.550040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.583937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518597013765820:2065] [node 6] ICP33 wake up from error state
2025-11-11T17:19:08.619671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518596933288346:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21010
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.637831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518596971447944:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:08.637893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518596971447945:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:08.637935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518596971447946:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:08.638541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518596971447965:2063] [node 4] ICP01 ready to work
2025-11-11T17:19:08.638838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518596971447966:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:08.658097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518596971447945:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17288947ns) {Configure:104(17285204ns) {ProcessConfigured:135(57189ns) {StartInitialHandshake:157(54472ns) {DropHandshakes:469(3605ns) {DropIncomingHandshake:441(758ns) DropOutgoingHandshake:456(860ns)} GenerateSessionVirtualId:988(1210ns) PrepareNewSessionHandshake:476(2163ns) {SwitchToState:178(705ns)}}}}}
2025-11-11T17:19:08.637176Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518597734393560:2066] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.620062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518596933288351:2069] [node 5] ICP01 ready to work
2025-11-11T17:19:08.584250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518597013765831:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.807627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518597630209403:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(334838636ns) {Configure:104(334834908ns) {ProcessConfigured:135(72070ns) {StartInitialHandshake:157(68737ns) {DropHandshakes:469(6368ns) {DropIncomingHandshake:441(849ns) DropOutgoingHandshake:456(480ns)} GenerateSessionVirtualId:988(807ns) PrepareNewSessionHandshake:476(2267ns) {SwitchToState:178(832ns)}}}}}
2025-11-11T17:19:08.637210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518597734393560:2066] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.620324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518596933288352:2071] [node 4] ICP01 ready to work
2025-11-11T17:19:08.781631Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.633325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518597013765834:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.668119Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address
2025-11-11T17:19:08.702732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518596933288351:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(82197134ns) {Configure:104(82192657ns) {ProcessConfigured:135(55060ns) {StartInitialHandshake:157(52552ns) {DropHandshakes:469(3866ns) {DropIncomingHandshake:441(1256ns) DropOutgoingHandshake:456(669ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:476(4902ns) {SwitchToState:178(647ns)}}}}}
2025-11-11T17:19:08.704201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571518596933288339:2061] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:08.704289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7571518596933288339:2061] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.704302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.704928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518596933288373:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21010
2025-11-11T17:19:08.704972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518596933288374:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21010
2025-11-11T17:19:08.765993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7571518596933288339:2061] [node 6] ICP33 wake up from error state
2025-11-11T17:19:08.652270Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address
2025-11-11T17:19:08.808768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518597630209414:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11289
2025-11-11T17:19:08.840180Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288374:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.808968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518597630209415:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11289
2025-11-11T17:19:08.809069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518597630209416:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11289
2025-11-11T17:19:08.809155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518597630209417:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11289
2025-11-11T17:19:08.824248Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 6] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address
2025-11-11T17:19:08.833504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518597630209461:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11289
2025-11-11T17:19:08.833660Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209416:2072] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.833739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518597630209402:2065] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:08.846655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518596933288339:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28421258ns) {Configure:104(28417924ns) {ProcessConfigured:135(6121633ns) {StartInitialHandshake:157(6118627ns) {DropHandshakes:469(3900ns) {DropIncomingHandshake:441(1017ns) DropOutgoingHandshake:456(604ns)} GenerateSessionVirtualId:988(1134ns) PrepareNewSessionHandshake:476(7278ns) {SwitchToState:178(1478ns)}}}}}
2025-11-11T17:19:08.846844Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288370:2090] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.846864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518596933288370:2090] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.813910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518597474255411:2062] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.635091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518597013765834:2073] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.847765Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209414:2070] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.847851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209415:2071] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.848369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518597630209382:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.849773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518597630209402:2065] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.849792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.854622Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209417:2073] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.861116Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209460:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.861135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518597630209460:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.677813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518597734393549:2060] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:08.853282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518596971448022:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
2025-11-11T17:19:08.814032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518597474255411:2062] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.635516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518597013765836:2074] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.853477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518596971448023:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
2025-11-11T17:19:08.872047Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288346:2067] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.814046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.677895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518597734393549:2060] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.850358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518597474255409:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(365270743ns) {Configure:104(365266543ns) {ProcessConfigured:135(7566287ns) {StartInitialHandshake:157(7563838ns) {DropHandshakes:469(1853ns) {DropIncomingHandshake:441(229ns) DropOutgoingHandshake:456(611ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:476(2465ns) {SwitchToState:178(1345ns)}}}}}
2025-11-11T17:19:08.636819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518597013765836:2074] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.899505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518597630209402:2065] [node 6] ICP35 event processing took too much time PendingConnection:278(16124210ns) {HandleHandshakeStatus:395(16121034ns) {LogHandshakeFail:472(10875ns) {UpdateErrorStateLog:544(5056ns)} DropOutgoingHandshake:456(591ns) ProcessPendingSessionEvents:495(1187922ns) {DropSessionEvent:512(7479ns) DropSessionEvent:512(3615ns) DropSessionEvent:512(3078ns) DropSessionEvent:512(2816ns) DropSessionEvent:512(9387ns) DropSessionEvent:512(3637ns) DropSessionEvent:512(3582ns) DropSessionEvent:512(3092ns) DropSessionEvent:512(1141772ns)} TransitToErrorState:752(82961ns) {SwitchToState:178(5920ns) ProcessPendingSessionEvents:495(7316ns) DropHandshakes:469(2580ns) {DropIncomingHandshake:441(234ns) DropOutgoingHandshake:456(645ns)}}}}
2025-11-11T17:19:08.850472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518597474255467:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.677913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:08.894166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288344:2066] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.859537Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518596971448023:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.894206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518596933288344:2066] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.899781Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518597630209460:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.850580Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518597474255418:2068] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.859683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518596971447966:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.685887Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518597734393560:2066] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:08.745879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518597734393593:2069] [node 4] ICP01 ready to work
2025-11-11T17:19:08.746004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518597734393549:2060] [node 6] ICP33 wake up from error state
2025-11-11T17:19:08.770197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518597734393537:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.770277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:08.770328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518597734393537:2050] [node 3] ICP22 created new session: [5:7571518597734393602:2050]
2025-11-11T17:19:08.770380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518597734393602:2050] [node 3] ICS09 handshake done sender: [5:7571518597734393560:2066] self: [5:7571518597734393559:0] peer: [3:7571518597013765837:0] socket: 238 qp: -1
2025-11-11T17:19:08.770413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518597734393602:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:08.770447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:08.802090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:308: Proxy [5:7571518597734393549:2060] [node 6] ICP35 event processing took too much time HoldByError:308(19874690ns) {WakeupFromErrorState:799(19872703ns) {SwitchToInitialState:201(5948ns) {SwitchToState:178(3555ns)}}}
2025-11-11T17:19:08.828437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518597734393604:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61689
2025-11-11T17:19:08.840883Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518597734393604:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.842155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518597734393535:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.858446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518597734393620:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61689
2025-11-11T17:19:08.858582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518597734393621:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61689
2025-11-11T17:19:08.858639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518597734393622:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61689
2025-11-11T17:19:08.859998Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518597734393620:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.860103Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518597734393615:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.860123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518597734393615:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.860811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518597734393536:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.865875Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518597734393615:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:08.914044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518597734393549:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.914088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:08.914139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518597734393549:2060] [node 6] ICP22 created new session: [5:7571518597734393626:2060]
2025-11-11T17:19:08.914187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518597734393626:2060] [node 6] ICS09 handshake done sender: [5:7571518597734393615:2113] self: [5:7571518597734393614:0] peer: [6:7571518596971448024:0] socket: 255 qp: -1
2025-11-11T17:19:08.914222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518597734393626:2060] [node 6] ICS10 traffic start
2025-11-11T17:19:08.914261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:08.942066Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:09.017547Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518597734393622:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.697526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518597013765848:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
2025-11-11T17:19:08.910571Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209410:2069] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.905862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518596933288351:2069] [node 5] ICP35 event processing took too much time PendingConnection:278(25630125ns) {IncomingHandshake:261(25627388ns) {DropIncomingHandshake:441(342ns) PrepareNewSessionHandshake:476(13679377ns) {SwitchToState:178(8467ns)} GenerateSessionVirtualId:988(1262ns) IssueIncomingHandshakeReply:193(12281ns)}}
2025-11-11T17:19:08.862120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518596971448025:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.851250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518597474255394:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.910607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518597630209410:2069] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.906135Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518596933288370:2090] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:08.906527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518596933288351:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.906588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:08.906630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518596933288351:2069] [node 5] ICP22 created new session: [1:7571518596933288406:2069]
2025-11-11T17:19:08.906672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518596933288406:2069] [node 5] ICS09 handshake done sender: [1:7571518596933288370:2090] self: [1:7571518596933288369:0] peer: [5:7571518597734393617:0] socket: 252 qp: -1
2025-11-11T17:19:08.906706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518596933288406:2069] [node 5] ICS10 traffic start
2025-11-11T17:19:08.906744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:08.917781Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518596933288344:2066] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:08.942071Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288372:2091] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.942098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518596933288372:2091] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.942589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518596933288319:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.942648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:08.942697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518596933288319:2048] [node 2] ICP22 created new session: [1:7571518596933288411:2048]
2025-11-11T17:19:08.942742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518596933288411:2048] [node 2] ICS09 handshake done sender: [1:7571518596933288344:2066] self: [1:7571518596933288338:0] peer: [2:7571518597630209467:0] socket: 223 qp: -1
2025-11-11T17:19:08.942777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518596933288411:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:08.942823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:08.944148Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518596933288372:2091] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:08.944387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518596933288352:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.944423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:08.944472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518596933288352:2071] [node 4] ICP22 created new session: [1:7571518596933288414:2071]
2025-11-11T17:19:08.944508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518596933288414:2071] [node 4] ICS09 handshake done sender: [1:7571518596933288372:2091] self: [1:7571518596933288371:0] peer: [4:7571518597474255479:0] socket: 251 qp: -1
2025-11-11T17:19:08.944540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518596933288414:2071] [node 4] ICS10 traffic start
2025-11-11T17:19:08.944567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:08.950634Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288337:2064] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.950659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518596933288337:2064] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.957849Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518596933288337:2064] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:08.958158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518596933288320:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.965785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:08.965837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518596933288320:2049] [node 3] ICP22 created new session: [1:7571518596933288418:2049]
2025-11-11T17:19:08.965877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518596933288418:2049] [node 3] ICS09 handshake done sender: [1:7571518596933288337:2064] self: [1:7571518596933288336:0] peer: [3:7571518597013765890:0] socket: 218 qp: -1
2025-11-11T17:19:08.965932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518596933288418:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:08.965984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:08.989919Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288373:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.000082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518596933288403:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.000105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518596933288403:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:09.001107Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518596933288403:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:09.001340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518596933288339:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:09.001408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:09.001451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518596933288339:2061] [node 6] ICP22 created new session: [1:7571518596933288422:2061]
2025-11-11T17:19:09.001496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518596933288422:2061] [node 6] ICS09 handshake done sender: [1:7571518596933288403:2118] self: [1:7571518596933288402:0] peer: [6:7571518596971448031:0] socket: 275 qp: -1
2025-11-11T17:19:09.001529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518596933288422:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:09.001561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:09.042018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518596933288320:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(83608694ns) {HandleHandshakeStatus:329(83605434ns) {DropIncomingHandshake:441(1286ns) DropOutgoingHandshake:456(1414ns) SwitchToState:178(672ns) ProcessPendingSessionEvents:495(51961589ns) {ForwardSessionEventToSession:612(51905769ns) ForwardSessionEventToSession:612(7536ns) ForwardSessionEventToSession:612(6500ns) ForwardSessionEventToSession:612(2461ns) ForwardSessionEventToSession:612(2646ns) ForwardSessionEventToSession:612(8804ns) ForwardSessionEventToSession:612(2548ns) ForwardSessionEventToSession:612(3169ns) ForwardSessionEventToSession:612(5206ns) ForwardSessionEventToSession:612(3143ns)}}}
2025-11-11T17:19:09.070646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518596933288339:2061] [node 6] ICP35 event processing took too much time PendingConnection:278(28510085ns) {HandleHandshakeStatus:329(28507982ns) {DropIncomingHandshake:441(9097ns) DropOutgoingHandshake:456(385ns) SwitchToState:178(549ns) ProcessPendingSessionEvents:495(8080404ns) {ForwardSessionEventToSession:612(67298ns) ForwardSessionEventToSession:612(8006566ns)}}}
2025-11-11T17:19:08.792586Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518597013765826:2068] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.045667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518597734393535:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518597734393604:2103] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.982264Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518596971448022:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.982534Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518596971448023:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:08.983019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518596971447966:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:08.983092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:08.983151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518596971447966:2066] [node 5] ICP22 created new session: [6:7571518596971448027:2066]
2025-11-11T17:19:08.983193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518596971448027:2066] [node 5] ICS09 handshake done sender: [6:7571518596971448023:2116] self: [6:7571518596971448024:0] peer: [5:7571518597734393614:0] socket: 261 qp: -1
2025-11-11T17:19:08.983241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518596971448027:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:08.983272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:08.983856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518596971447946:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.984161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518596971448026:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
2025-11-11T17:19:08.993838Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518596971448026:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.993983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518596971447944:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:09.022991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518601266415328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
2025-11-11T17:19:09.023119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518601266415329:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
2025-11-11T17:19:09.046167Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518596971447976:2074] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.046193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [6:7571518596971447976:2074] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:09.065896Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518596971448026:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:09.066254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518596971447944:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.066336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:09.066382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518596971447944:2048] [node 1] ICP22 created new session: [6:7571518601266415330:2048]
2025-11-11T17:19:09.066427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518601266415330:2048] [node 1] ICS09 handshake done sender: [6:7571518596971448026:2118] self: [6:7571518596971448031:0] peer: [1:7571518596933288402:0] socket: 226 qp: -1
2025-11-11T17:19:09.066459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518601266415330:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:09.066496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:09.084071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518596971447976:2074] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:08.871045Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518597474255419:2069] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.792662Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518597013765834:2073] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:08.918395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518597630209402:2065] [node 6] ICP33 wake up from error state
2025-11-11T17:19:09.114127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7571518596971447965:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.871184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518597474255393:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:08.908240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518597474255476:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:08.917774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518597474255476:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.917832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518597474255411:2062] [node 6] ICP33 wake up from error state
2025-11-11T17:19:08.926323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518597474255392:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.926410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518597474255477:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:08.926474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518597474255478:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:08.961981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518597474255480:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:08.966553Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518597474255418:2068] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:08.966752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518597474255419:2069] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:08.966934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518597474255394:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:08.967004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:08.967052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518597474255394:2050] [node 3] ICP22 created new session: [4:7571518597474255481:2050]
2025-11-11T17:19:08.967114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518597474255481:2050] [node 3] ICS09 handshake done sender: [4:7571518597474255418:2068] self: [4:7571518597474255473:0] peer: [3:7571518597013765822:0] socket: 231 qp: -1
2025-11-11T17:19:08.967148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518597474255481:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:08.967187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:08.967615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518597474255393:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:08.967700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:08.967762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518597474255393:2049] [node 2] ICP22 created new session: [4:7571518597474255483:2049]
2025-11-11T17:19:08.967805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518597474255483:2049] [node 2] ICS09 handshake done sender: [4:7571518597474255419:2069] self: [4:7571518597474255475:0] peer: [2:7571518597630209409:0] socket: 232 qp: -1
2025-11-11T17:19:08.967837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518597474255483:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:08.967884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:09.006074Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518597474255467:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.006398Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518597474255476:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:09.034074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518597474255392:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.034170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:09.034222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518597474255392:2048] [node 1] ICP22 created new session: [4:7571518601769222782:2048]
2025-11-11T17:19:09.034287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518601769222782:2048] [node 1] ICS09 handshake done sender: [4:7571518597474255476:2118] self: [4:7571518597474255479:0] peer: [1:7571518596933288371:0] socket: 267 qp: -1
2025-11-11T17:19:09.034326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518601769222782:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:09.034385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:09.035722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518597474255411:2062] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:09.070321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518601769222785:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:09.076526Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518601769222785:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.077687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518597474255472:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.077725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518597474255472:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:09.097725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518597474255472:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:09.110172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518597474255409:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:09.110269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:09.110322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518597474255409:2061] [node 5] ICP22 created new session: [4:7571518601769222789:2061]
2025-11-11T17:19:09.110365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518601769222789:2061] [node 5] ICS09 handshake done sender: [4:7571518597474255472:2117] self: [4:7571518597474255470:0] peer: [5:7571518602029360926:0] socket: 259 qp: -1
2025-11-11T17:19:09.110406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518601769222789:2061] [node 5] ICS10 traffic start
2025-11-11T17:19:09.110464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:09.054044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518597734393593:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(282763262ns) {Configure:104(282757453ns) {ProcessConfigured:135(48127749ns) {StartInitialHandshake:157(48123851ns) {DropHandshakes:469(2543ns) {DropIncomingHandshake:441(626ns) DropOutgoingHandshake:456(370ns)} GenerateSessionVirtualId:988(605ns) PrepareNewSessionHandshake:476(5026ns) {SwitchToState:178(1813ns)}}}}}
2025-11-11T17:19:09.157683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518601769222788:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28004
2025-11-11T17:19:08.793183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518597013765836:2074] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.054550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518597734393625:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61689
2025-11-11T17:19:09.055083Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518597734393604:2103] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:09.055473Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518597734393620:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:09.055705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518597734393593:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:09.055802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518597734393536:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.055866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:09.055910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518597734393536:2049] [node 2] ICP22 created new session: [5:7571518602029360927:2049]
2025-11-11T17:19:09.055962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518602029360927:2049] [node 2] ICS09 handshake done sender: [5:7571518597734393620:2116] self: [5:7571518597734393624:0] peer: [2:7571518597630209459:0] socket: 257 qp: -1
2025-11-11T17:19:09.056000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518602029360927:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:09.056035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:09.081818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518597734393535:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(44164185ns) {HandleHandshakeStatus:395(44162228ns) {LogHandshakeFail:472(16045920ns) {UpdateErrorStateLog:544(10570ns)} DropOutgoingHandshake:456(1619ns) ProcessPendingSessionEvents:495(16048399ns) {DropSessionEvent:512(11789ns) DropSessionEvent:512(5317ns) DropSessionEvent:512(5874ns) DropSessionEvent:512(25127ns) DropSessionEvent:512(5393ns) DropSessionEvent:512(4569ns) DropSessionEvent:512(3491ns) DropSessionEvent:512(3776ns) DropSessionEvent:512(4434ns) DropSessionEvent:512(4400ns)}}}
2025-11-11T17:19:09.081880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518597734393535:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.081923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:09.081967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518597734393535:2048] [node 1] ICP22 created new session: [5:7571518602029360930:2048]
2025-11-11T17:19:09.082011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518602029360930:2048] [node 1] ICS09 handshake done sender: [5:7571518597734393604:2103] self: [5:7571518597734393617:0] peer: [1:7571518596933288369:0] socket: 234 qp: -1
2025-11-11T17:19:09.082041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518602029360930:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:09.082074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:09.135180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518602029360932:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61689
2025-11-11T17:19:08.918972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518597630209472:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11289
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:09.167188Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518597474255467:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:08.793233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:08.793279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518597013765836:2074] [node 5] ICP22 created new session: [3:7571518597013765877:2074]
2025-11-11T17:19:08.793319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518597013765877:2074] [node 5] ICS09 handshake done sender: [3:7571518597013765834:2073] self: [3:7571518597013765837:0] peer: [5:7571518597734393559:0] socket: 239 qp: -1
2025-11-11T17:19:08.793359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518597013765877:2074] [node 5] ICS10 traffic start
2025-11-11T17:19:08.793394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:08.870214Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518597013765831:2070] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.901864Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518597013765823:2066] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.901905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518597013765823:2066] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.945847Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518597013765823:2066] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:08.950006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518597013765800:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(319365516ns) {Configure:104(319359960ns) {ProcessConfigured:135(7927976ns) {StartInitialHandshake:157(7925014ns) {DropHandshakes:469(3579ns) {DropIncomingHandshake:441(645ns) DropOutgoingHandshake:456(664ns)} GenerateSessionVirtualId:988(1287ns) PrepareNewSessionHandshake:476(3754ns) {SwitchToState:178(1826ns)}}}}}
2025-11-11T17:19:08.950250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518597013765800:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:08.997927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518597013765850:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:08.997953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518597013765850:2086] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:08.998475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518597013765819:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.998575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:08.998647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518597013765819:2063] [node 4] ICP22 created new session: [3:7571518597013765893:2063]
2025-11-11T17:19:08.998690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518597013765893:2063] [node 4] ICS09 handshake done sender: [3:7571518597013765823:2066] self: [3:7571518597013765822:0] peer: [4:7571518597474255473:0] socket: 221 qp: -1
2025-11-11T17:19:08.998724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518597013765893:2063] [node 4] ICS10 traffic start
2025-11-11T17:19:08.998768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:09.016912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518597013765891:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
2025-11-11T17:19:09.109772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518597013765892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
2025-11-11T17:19:09.114029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518597013765826:2068] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:09.114361Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518597013765850:2086] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:09.114631Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518597013765892:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.114664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518597013765800:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.114738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:09.114779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518597013765800:2048] [node 1] ICP22 created new session: [3:7571518601308733193:2048]
2025-11-11T17:19:09.114821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518601308733193:2048] [node 1] ICS09 handshake done sender: [3:7571518597013765826:2068] self: [3:7571518597013765890:0] peer: [1:7571518596933288336:0] socket: 220 qp: -1
2025-11-11T17:19:09.114850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518601308733193:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:09.114903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:09.115689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518597013765820:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:09.115752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:09.115790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518597013765820:2065] [node 6] ICP22 created new session: [3:7571518601308733195:2065]
2025-11-11T17:19:09.115831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518601308733195:2065] [node 6] ICS09 handshake done sender: [3:7571518597013765850:2086] self: [3:7571518597013765849:0] peer: [6:7571518596971448029:0] socket: 253 qp: -1
2025-11-11T17:19:09.115856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518601308733195:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:09.115879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:09.116342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518597013765820:2065] [node 6] ICP18 handshake (actor [3:7571518597013765892:2121]) is too old
2025-11-11T17:19:08.958069Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518597630209410:2069] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:09.157802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518601228255720:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21010
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:09.181980Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25709) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:09.182151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518601308733197:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
2025-11-11T17:19:09.183270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518597734393593:2069] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25709) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518597734393622:2118] held: no
2025-11-11T17:19:09.183393Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518597734393622:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:09.168612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:08.973767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518597630209403:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:08.973848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:08.973909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518597630209403:2066] [node 5] ICP22 created new session: [2:7571518597630209476:2066]
2025-11-11T17:19:08.973953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518597630209476:2066] [node 5] ICS09 handshake done sender: [2:7571518597630209460:2115] self: [2:7571518597630209459:0] peer: [5:7571518597734393624:0] socket: 256 qp: -1
2025-11-11T17:19:08.973984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518597630209476:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:08.974018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:08.974463Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:09.083723Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518597630209414:2070] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:09.093870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518597630209382:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518597630209414:2070] held: no
2025-11-11T17:19:09.117846Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209461:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.117880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518597630209404:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:09.117975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:09.118018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518597630209404:2064] [node 4] ICP22 created new session: [2:7571518601925176774:2064]
2025-11-11T17:19:09.118062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518601925176774:2064] [node 4] ICS09 handshake done sender: [2:7571518597630209410:2069] self: [2:7571518597630209409:0] peer: [4:7571518597474255475:0] socket: 229 qp: -1
2025-11-11T17:19:09.118090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518601925176774:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:09.118139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:09.145882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518597630209382:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(29757522ns) {HandleHandshakeStatus:395(29754934ns) {LogHandshakeFail:472(13318ns) {UpdateErrorStateLog:544(5691ns)} DropOutgoingHandshake:456(4555ns) ProcessPendingSessionEvents:495(15038286ns) {DropSessionEvent:512(19621ns) DropSessionEvent:512(4047ns) DropSessionEvent:512(4374ns) DropSessionEvent:512(3269ns) DropSessionEvent:512(3552ns) DropSessionEvent:512(7135935ns) DropSessionEvent:512(4902ns) DropSessionEvent:512(7843301ns) DropSessionEvent:512(5126ns) DropSessionEvent:512(5154ns)}}}
2025-11-11T17:19:09.167429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518597630209382:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.167513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:09.167555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518597630209382:2048] [node 1] ICP22 created new session: [2:7571518601925176778:2048]
2025-11-11T17:19:09.167597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518601925176778:2048] [node 1] ICS09 handshake done sender: [2:7571518597630209414:2070] self: [2:7571518597630209467:0] peer: [1:7571518596933288338:0] socket: 230 qp: -1
2025-11-11T17:19:09.167629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518601925176778:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:09.167682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:09.182948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518597474255411:2062] [node 6] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:09.158298Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:09.168672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518596971447965:2063] [node 4] ICP22 created new session: [6:7571518601266415333:2063]
2025-11-11T17:19:09.205814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:09.205887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518597474255411:2062] [node 6] ICP22 created new session: [4:7571518601769222791:2062]
2025-11-11T17:19:09.205931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518601769222791:2062] [node 6] ICS09 handshake done sender: [4:7571518597474255467:2114] self: [4:7571518601769222784:0] peer: [6:7571518596971447975:0] socket: 254 qp: -1
2025-11-11T17:19:09.205960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518601769222791:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:09.206013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:09.206211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518597474255411:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(23198116ns) {HandleHandshakeStatus:329(23194492ns) {DropIncomingHandshake:441(1229ns) DropOutgoingHandshake:456(596ns) SwitchToState:178(597ns) ProcessPendingSessionEvents:495(1198ns)}}
2025-11-11T17:19:09.201851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518597734393593:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.166579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518601228255721:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21010
2025-11-11T17:19:09.210364Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209475:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.210396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518597630209475:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:09.222378Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518597630209463:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.222398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518597630209463:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:09.166637Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:09.201911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:09.221828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518601308733197:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.173789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518601266415333:2063] [node 4] ICS09 handshake done sender: [6:7571518596971447976:2074] self: [6:7571518596971447975:0] peer: [4:7571518601769222784:0] socket: 248 qp: -1
2025-11-11T17:19:09.226473Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518597630209463:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:09.201954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518597734393593:2069] [node 4] ICP22 created new session: [5:7571518602029360933:2069]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:09.201996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518602029360933:2069] [node 4] ICS09 handshake done sender: [5:7571518597734393622:2118] self: [5:7571518602029360926:0] peer: [4:7571518597474255470:0] socket: 263 qp: -1
2025-11-11T17:19:09.202034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518602029360933:2069] [node 4] ICS10 traffic start
2025-11-11T17:19:09.202073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:09.173837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518601266415333:2063] [node 4] ICS10 traffic start
2025-11-11T17:19:09.222005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518597013765801:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:09.238081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518601308733199:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40454
2025-11-11T17:19:09.266325Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:09.267846Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518601308733197:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:09.246773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518597630209383:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:09.173874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:09.174368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518596971447965:2063] [node 4] ICP35 event processing took too much time PendingConnection:278(59986556ns) {HandleHandshakeStatus:329(59985051ns) {DropIncomingHandshake:441(764ns) DropOutgoingHandshake:456(751ns) SwitchToState:178(2975ns) ProcessPendingSessionEvents:495(105873ns) {ForwardSessionEventToSession:612(49406ns) ForwardSessionEventToSession:612(4017ns) ForwardSessionEventToSession:612(5609ns) ForwardSessionEventToSession:612(2773ns) ForwardSessionEventToSession:612(4390ns) ForwardSessionEventToSession:612(4523ns) ForwardSessionEventToSession:612(1778ns) ForwardSessionEventToSession:612(2776ns) ForwardSessionEventToSession:612(4377ns) ForwardSessionEventToSession:612(15236ns)}}}
2025-11-11T17:19:09.174488Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1694) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:09.175846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518596971447946:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1694) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518596971448022:2115] held: no
2025-11-11T17:19:09.176378Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518596971448022:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:09.196691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518596971447946:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.206801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518601266415335:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
2025-11-11T17:19:09.209957Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518601266415335:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:09.210075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518596971447945:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:09.223763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:09.241700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518596971447946:2050] [node 3] ICP22 created new session: [6:7571518601266415337:2050]
2025-11-11T17:19:09.241795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518601266415337:2050] [node 3] ICS09 handshake done sender: [6:7571518596971448022:2115] self: [6:7571518596971448029:0] peer: [3:7571518597013765849:0] socket: 260 qp: -1
2025-11-11T17:19:09.241841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518601266415337:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:09.241928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:09.242225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518596971447946:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(58479161ns) {HandleHandshakeStatus:329(58476941ns) {DropIncomingHandshake:441(2178ns) DropOutgoingHandshake:456(577ns) SwitchToState:178(878ns) ProcessPendingSessionEvents:495(1095ns)}}
2025-11-11T17:19:09.253280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518601266415339:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55581
2025-11-11T17:19:09.261009Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518601266415335:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:09.261359Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:09.262602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518596971447945:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.262696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:09.262743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518596971447945:2049] [node 2] ICP22 created new session: [6:7571518601266415340:2049]
2025-11-11T17:19:09.262804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518601266415340:2049] [node 2] ICS09 handshake done sender: [6:7571518601266415335:2121] self: [6:7571518601266415336:0] peer: [2:7571518597630209474:0] socket: 233 qp: -1
2025-11-11T17:19:09.262849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518601266415340:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:09.262885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:09.263181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518596971447945:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:09.251969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:09.252030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518597630209383:2049] [node 3] ICP22 created new session: [2:7571518601925176783:2049]
2025-11-11T17:19:09.252092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518601925176783:2049] [node 3] ICS09 handshake done sender: [2:7571518597630209463:2117] self: [2:7571518597630209462:0] peer: [3:7571518601308733198:0] socket: 279 qp: -1
2025-11-11T17:19:09.252132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518601925176783:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:09.252205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:09.252773Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518597630209475:2121] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:09.254784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518597630209402:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:09.254871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:09.254915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518597630209402:2065] [node 6] ICP22 created new session: [2:7571518601925176785:2065]
2025-11-11T17:19:09.254976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518601925176785:2065] [node 6] ICS09 handshake done sender: [2:7571518597630209475:2121] self: [2:7571518597630209474:0] peer: [6:7571518601266415336:0] socket: 224 qp: -1
2025-11-11T17:19:09.255008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518601925176785:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:09.255042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:09.279079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518597013765801:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518601308733197:2122] held: no
2025-11-11T17:19:09.279551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518597013765801:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:09.279644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:09.279689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518597013765801:2049] [node 2] ICP22 created new session: [3:7571518601308733200:2049]
2025-11-11T17:19:09.279751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518601308733200:2049] [node 2] ICS09 handshake done sender: [3:7571518601308733197:2122] self: [3:7571518601308733198:0] peer: [2:7571518597630209462:0] socket: 284 qp: -1
2025-11-11T17:19:09.279787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518601308733200:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:09.279825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:09.329742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:09.329978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518601925176778:2048] [node 1] ICS01 socket: 230 reason# EndOfStream
2025-11-11T17:19:09.330018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518597630209382:2048] [node 1] ICP30 unregister session Session# [2:7571518601925176778:2048] VirtualId# [2:7571518597630209467:0]
2025-11-11T17:19:09.330053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518601925176778:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.330141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:09.330183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:09.341742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:09.342000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518601308733193:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-11T17:19:09.342040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518597013765800:2048] [node 1] ICP30 unregister session Session# [3:7571518601308733193:2048] VirtualId# [3:7571518597013765890:0]
2025-11-11T17:19:09.342070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518601308733193:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.342183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:09.342223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:09.353864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:09.353999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:09.354222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518601266415330:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-11-11T17:19:09.354256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518596971447944:2048] [node 1] ICP30 unregister session Session# [6:7571518601266415330:2048] VirtualId# [6:7571518596971448031:0]
2025-11-11T17:19:09.354289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518601266415330:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.354378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:09.354416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:09.354492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:09.354677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518601769222782:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-11-11T17:19:09.354703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518597474255392:2048] [node 1] ICP30 unregister session Session# [4:7571518601769222782:2048] VirtualId# [4:7571518597474255479:0]
2025-11-11T17:19:09.354725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518601769222782:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.354770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:09.354800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:09.355422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518602029360930:2048] [node 1] ICS01 socket: 234 reason# EndOfStream
2025-11-11T17:19:09.355466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518597734393535:2048] [node 1] ICP30 unregister session Session# [5:7571518602029360930:2048] VirtualId# [5:7571518597734393617:0]
2025-11-11T17:19:09.355486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518602029360930:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.355540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:09.355572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:09.441752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:09.441983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518601308733200:2049] [node 2] ICS01 socket: 284 reason# EndOfStream
2025-11-11T17:19:09.442053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518597013765801:2049] [node 2] ICP30 unregister session Session# [3:7571518601308733200:2049] VirtualId# [3:7571518601308733198:0]
2025-11-11T17:19:09.442108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518601308733200:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.442245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:09.442298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:09.453348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:09.453579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518602029360927:2049] [node 2] ICS01 socket: 257 reason# EndOfStream
2025-11-11T17:19:09.453634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518597734393536:2049] [node 2] ICP30 unregister session Session# [5:7571518602029360927:2049] VirtualId# [5:7571518597734393624:0]
2025-11-11T17:19:09.453667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518602029360927:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.453744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:09.453792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:09.453853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:09.454034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518601266415340:2049] [node 2] ICS01 socket: 233 reason# EndOfStream
2025-11-11T17:19:09.454061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518596971447945:2049] [node 2] ICP30 unregister session Session# [6:7571518601266415340:2049] VirtualId# [6:7571518601266415336:0]
2025-11-11T17:19:09.454081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518601266415340:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.454139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:09.454179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:09.453771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:09.453984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518597474255483:2049] [node 2] ICS01 socket: 232 reason# EndOfStream
2025-11-11T17:19:09.454025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518597474255393:2049] [node 2] ICP30 unregister session Session# [4:7571518597474255483:2049] VirtualId# [4:7571518597474255475:0]
2025-11-11T17:19:09.454050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518597474255483:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.454145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:09.454185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:09.541960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:09.542207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518601266415337:2050] [node 3] ICS01 socket: 260 reason# EndOfStream
2025-11-11T17:19:09.542253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518596971447946:2050] [node 3] ICP30 unregister session Session# [6:7571518601266415337:2050] VirtualId# [6:7571518596971448029:0]
2025-11-11T17:19:09.542285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518601266415337:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.542378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:09.542419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:09.543883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:09.544100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518597474255481:2050] [node 3] ICS01 socket: 231 reason# EndOfStream
2025-11-11T17:19:09.544133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518597474255394:2050] [node 3] ICP30 unregister session Session# [4:7571518597474255481:2050] VirtualId# [4:7571518597474255473:0]
2025-11-11T17:19:09.544161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518597474255481:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.544239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:09.544288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:09.542498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:09.542698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518597734393602:2050] [node 3] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-11T17:19:09.542724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518597734393602:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.542780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:09.542817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518597734393602:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:09.543876Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1694) Couldn't connect to any resolved address
2025-11-11T17:19:09.544113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518597734393537:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1694) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:09.544167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518597734393602:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:09.544193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518597734393537:2050] [node 3] ICP30 unregister session Session# [5:7571518597734393602:2050] VirtualId# [5:7571518597734393559:0]
2025-11-11T17:19:09.544234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:09.544263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518597734393537:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1694) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:09.544276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1694) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:09.563842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518597734393537:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:09.599002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:09.599260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518601266415333:2063] [node 4] ICS01 socket: 248 reason# EndOfStream
2025-11-11T17:19:09.599299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518596971447965:2063] [node 4] ICP30 unregister session Session# [6:7571518601266415333:2063] VirtualId# [6:7571518596971447975:0]
2025-11-11T17:19:09.599331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518601266415333:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.599402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:09.599450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:09.599528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:09.599717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518602029360933:2069] [node 4] ICS01 socket: 263 reason# EndOfStream
2025-11-11T17:19:09.599748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518597734393593:2069] [node 4] ICP30 unregister session Session# [5:7571518602029360933:2069] VirtualId# [5:7571518602029360926:0]
2025-11-11T17:19:09.599769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518602029360933:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.599809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:09.599842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:09.645888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:09.646135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518596971448027:2066] [node 5] ICS01 socket: 261 reason# EndOfStream
2025-11-11T17:19:09.646176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518596971447966:2066] [node 5] ICP30 unregister session Session# [6:7571518596971448027:2066] VirtualId# [6:7571518596971448024:0]
2025-11-11T17:19:09.646208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518596971448027:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:09.646316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:09.646356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:09.842030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518600727598957:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:09.842075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518600727598958:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:09.884935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518600931710418:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:09.884978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518600931710419:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:09.948905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518601046860563:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:09.948949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518601046860564:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:09.993889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518601753158470:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:09.993928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518601753158471:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:09.993954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518601753158472:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:10.055190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518606834925161:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:10.055230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518606834925162:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:10.055257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518606834925163:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:10.121921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518604211656203:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:10.121962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518604211656204:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:10.121989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518604211656205:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:10.445990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518604211656219:2059] [node 5] ICP01 ready to work
2025-11-11T17:19:10.465480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518604211656225:2066] [node 4] ICP01 ready to work
2025-11-11T17:19:10.477734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518605226677773:2075] [node 4] ICP01 ready to work
2025-11-11T17:19:10.465800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518605022566315:2105] [node 4] ICP01 ready to work
2025-11-11T17:19:10.540098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518606048125791:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:10.484809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518605341827875:2058] [node 4] ICP01 ready to work
2025-11-11T17:19:10.540364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518606834925195:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:10.539672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518605226677774:2077] [node 6] ICP01 ready to work
2025-11-11T17:19:10.492081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518605022566317:2106] [node 6] ICP01 ready to work
2025-11-11T17:19:10.599142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518605341827919:2065] [node 5] ICP01 ready to work
2025-11-11T17:19:10.579965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518605226677778:2076] [node 5] ICP01 ready to work
2025-11-11T17:19:10.513729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518605022566318:2107] [node 5] ICP01 ready to work
2025-11-11T17:19:10.661770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518606048125830:2071] [node 6] ICP01 ready to work
2025-11-11T17:19:10.615512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518606834925163:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(147536772ns) {Configure:104(147533074ns) {ProcessConfigured:135(71569ns) {StartInitialHandshake:157(68267ns) {DropHandshakes:469(2577ns) {DropIncomingHandshake:441(581ns) DropOutgoingHandshake:456(487ns)} GenerateSessionVirtualId:988(789ns) PrepareNewSessionHandshake:476(2022ns) {SwitchToState:178(861ns)}}}}}
2025-11-11T17:19:10.631184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518606834925162:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(150688011ns) {Configure:104(150685733ns) {ProcessConfigured:135(44436ns) {StartInitialHandshake:157(42959ns) {DropHandshakes:469(884ns) {DropIncomingHandshake:441(247ns) DropOutgoingHandshake:456(284ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:476(1111ns) {SwitchToState:178(608ns)}}}}}
2025-11-11T17:19:10.653604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518605341827933:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:10.691116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518600727598957:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(131388530ns) {Configure:104(131383206ns) {ProcessConfigured:135(59916ns) {StartInitialHandshake:157(56729ns) {DropHandshakes:469(2517ns) {DropIncomingHandshake:441(548ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(1165ns) PrepareNewSessionHandshake:476(1247ns) {SwitchToState:178(430ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.683124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518600931710418:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(65308179ns) {Configure:104(65296815ns) {ProcessConfigured:135(64199ns) {StartInitialHandshake:157(61372ns) {DropHandshakes:469(2231ns) {DropIncomingHandshake:441(520ns) DropOutgoingHandshake:456(400ns)} GenerateSessionVirtualId:988(1239ns) PrepareNewSessionHandshake:476(1528ns) {SwitchToState:178(502ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.713628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518606834925230:2091] [node 6] ICP01 ready to work
2025-11-11T17:19:10.698758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518600727598958:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(220555361ns) {Configure:104(220550933ns) {ProcessConfigured:135(54267ns) {StartInitialHandshake:157(51430ns) {DropHandshakes:469(1812ns) {DropIncomingHandshake:441(438ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:476(1701ns) {SwitchToState:178(568ns)}}}}}
2025-11-11T17:19:10.699204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518605022566319:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3089
2025-11-11T17:19:10.714376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518605022566317:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21624723ns) {Configure:104(21619621ns) {ProcessConfigured:135(19351857ns) {StartInitialHandshake:157(19349182ns) {DropHandshakes:469(1693ns) {DropIncomingHandshake:441(415ns) DropOutgoingHandshake:456(400ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:476(3330ns) {SwitchToState:178(1041ns)}}}}}
2025-11-11T17:19:10.714892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518605022566331:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3089
2025-11-11T17:19:10.721190Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566319:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.706471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518605226677786:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17453
2025-11-11T17:19:10.769550Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566331:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.751636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518605341827933:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(75842119ns) {Configure:104(75834134ns) {ProcessConfigured:135(62903ns) {StartInitialHandshake:157(58395ns) {DropHandshakes:469(2729ns) {DropIncomingHandshake:441(559ns) DropOutgoingHandshake:456(579ns)} GenerateSessionVirtualId:988(579ns) PrepareNewSessionHandshake:476(1547ns) {SwitchToState:178(526ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.778780Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566326:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.778809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518605022566326:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.779861Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518605022566326:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:10.774322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518606834925195:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(171529522ns) {Configure:104(171523938ns) {ProcessConfigured:135(59029ns) {StartInitialHandshake:157(55237ns) {DropHandshakes:469(1532ns) {DropIncomingHandshake:441(251ns) DropOutgoingHandshake:456(167ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:476(1330ns) {SwitchToState:178(443ns)}}}}}
2025-11-11T17:19:10.776380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518606834925229:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:10.776527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518606834925233:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:10.776681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518606834925234:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:10.706996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518605226677787:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17453
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.758095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518604211656225:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(275297838ns) {Configure:104(275291653ns) {ProcessConfigured:135(45409ns) {StartInitialHandshake:157(40621ns) {DropHandshakes:469(1521ns) {DropIncomingHandshake:441(420ns) DropOutgoingHandshake:456(309ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:476(1145ns) {SwitchToState:178(501ns)}}}}}
2025-11-11T17:19:10.801640Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518606834925233:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.780255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518600727598958:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:10.730997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518600931710419:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(114403049ns) {Configure:104(114397030ns) {ProcessConfigured:135(59245ns) {StartInitialHandshake:157(57010ns) {DropHandshakes:469(2259ns) {DropIncomingHandshake:441(470ns) DropOutgoingHandshake:456(379ns)} GenerateSessionVirtualId:988(904ns) PrepareNewSessionHandshake:476(1505ns) {SwitchToState:178(611ns)}}}}}
2025-11-11T17:19:10.792014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518604211656204:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(350696594ns) {Configure:104(350693326ns) {ProcessConfigured:135(21093496ns) {StartInitialHandshake:157(21091384ns) {DropHandshakes:469(516ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(177ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(2488ns) {SwitchToState:178(781ns)}}}}}
2025-11-11T17:19:10.792796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656281:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:10.792842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656282:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:10.813699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518604211656203:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(343114907ns) {Configure:104(343110608ns) {ProcessConfigured:135(15887631ns) {StartInitialHandshake:157(15884321ns) {DropHandshakes:469(877ns) {DropIncomingHandshake:441(218ns) DropOutgoingHandshake:456(239ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:476(4461ns) {SwitchToState:178(2356ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.816567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656283:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.751963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518605341827934:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.801860Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518606834925234:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.849863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518604211656282:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.849988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518604211656203:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.753662Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518605226677786:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.787963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518605226677787:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.805999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518605226677774:2077] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(188033371ns) {Configure:104(164145466ns) {ProcessConfigured:135(42871728ns) {StartInitialHandshake:157(35215028ns) {DropHandshakes:469(1651ns) {DropIncomingHandshake:441(227ns) DropOutgoingHandshake:456(252ns)} GenerateSessionVirtualId:988(1424ns) PrepareNewSessionHandshake:476(3234ns) {SwitchToState:178(953ns)}}}}}
2025-11-11T17:19:10.806175Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518605226677783:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.806196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518605226677783:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.806789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518600931710418:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.809797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518605226677783:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:10.838001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518605226677799:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17453
2025-11-11T17:19:10.778381Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518605341827934:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.780302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:10.802007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518606834925162:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.838981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518606048125830:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(79094125ns) {Configure:104(79091473ns) {ProcessConfigured:135(45194878ns) {StartInitialHandshake:157(45193585ns) {DropHandshakes:469(1244ns) {DropIncomingHandshake:441(346ns) DropOutgoingHandshake:456(270ns)} GenerateSessionVirtualId:988(449ns) PrepareNewSessionHandshake:476(2743ns) {SwitchToState:178(500ns)}}}}}
2025-11-11T17:19:10.879455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518605226677778:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:10.879526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:10.879569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518605226677778:2076] [node 5] ICP22 created new session: [2:7571518605226677800:2076]
2025-11-11T17:19:10.879606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518605226677800:2076] [node 5] ICS09 handshake done sender: [2:7571518605226677783:2111] self: [2:7571518605226677782:0] peer: [5:7571518606834925240:0] socket: 242 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.881411Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518605226677787:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:10.885755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518605226677800:2076] [node 5] ICS10 traffic start
2025-11-11T17:19:10.887891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518600931710418:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:10.887932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:10.887969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518600931710418:2048] [node 1] ICP22 created new session: [2:7571518605226677803:2048]
2025-11-11T17:19:10.888003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518605226677803:2048] [node 1] ICS09 handshake done sender: [2:7571518605226677787:2114] self: [2:7571518605226677795:0] peer: [1:7571518605022566320:0] socket: 240 qp: -1
2025-11-11T17:19:10.888028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518605226677803:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:10.888076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:10.889655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.778511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518601046860563:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.780339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518600727598958:2049] [node 3] ICP22 created new session: [1:7571518605022566337:2049]
2025-11-11T17:19:10.889362Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518604211656281:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.897350Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518604211656283:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.856935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518601753158470:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(355814024ns) {Configure:104(355808929ns) {ProcessConfigured:135(24820204ns) {StartInitialHandshake:157(24817755ns) {DropHandshakes:469(2334ns) {DropIncomingHandshake:441(430ns) DropOutgoingHandshake:456(432ns)} GenerateSessionVirtualId:988(808ns) PrepareNewSessionHandshake:476(2812ns) {SwitchToState:178(1252ns)}}}}}
2025-11-11T17:19:10.819258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518606834925241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:10.779666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518605341827938:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.861982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518606834925161:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(183367423ns) {Configure:104(183360545ns) {ProcessConfigured:135(68781ns) {StartInitialHandshake:157(65482ns) {DropHandshakes:469(2545ns) {DropIncomingHandshake:441(805ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(967ns) PrepareNewSessionHandshake:476(1248ns) {SwitchToState:178(502ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.780379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518605022566337:2049] [node 3] ICS09 handshake done sender: [1:7571518605022566326:2113] self: [1:7571518605022566325:0] peer: [3:7571518605341827937:0] socket: 237 qp: -1
2025-11-11T17:19:10.898776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518604211656219:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.781183Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518605341827934:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.905626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518605226677798:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17453
2025-11-11T17:19:10.941615Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:10.909618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518604211656205:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.897176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518601753158472:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(388047970ns) {Configure:104(388042726ns) {ProcessConfigured:135(43367816ns) {StartInitialHandshake:157(43366114ns) {DropHandshakes:469(844ns) {DropIncomingHandshake:441(204ns) DropOutgoingHandshake:456(152ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:476(3055ns) {SwitchToState:178(1294ns)}}}}}
2025-11-11T17:19:10.780405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518605022566337:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:10.957389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518605226677778:2076] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:10.780431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:10.798093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518605022566315:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(202619128ns) {Configure:104(202613233ns) {ProcessConfigured:135(23064708ns) {StartInitialHandshake:157(23062071ns) {DropHandshakes:469(2367ns) {DropIncomingHandshake:441(538ns) DropOutgoingHandshake:456(659ns)} GenerateSessionVirtualId:988(1019ns) PrepareNewSessionHandshake:476(2831ns) {SwitchToState:178(1558ns)}}}}}
2025-11-11T17:19:10.819614Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566321:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.819636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518605022566321:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.833916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518605022566321:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:10.841732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518605022566340:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3089
2025-11-11T17:19:10.878166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566340:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.878275Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566323:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.878296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518605022566323:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.881996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518600727598957:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:10.882071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:10.889527Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518605022566323:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:10.893696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518600727598957:2048] [node 2] ICP22 created new session: [1:7571518605022566345:2048]
2025-11-11T17:19:10.897636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566330:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.897675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518605022566330:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.905700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518605022566345:2048] [node 2] ICS09 handshake done sender: [1:7571518605022566321:2111] self: [1:7571518605022566320:0] peer: [2:7571518605226677795:0] socket: 235 qp: -1
2025-11-11T17:19:10.914705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518605022566345:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:10.914746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:10.930475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518605022566318:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:10.930517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:10.930565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518605022566318:2107] [node 5] ICP22 created new session: [1:7571518605022566347:2107]
2025-11-11T17:19:10.930603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518605022566347:2107] [node 5] ICS09 handshake done sender: [1:7571518605022566323:2112] self: [1:7571518605022566322:0] peer: [5:7571518606834925244:0] socket: 236 qp: -1
2025-11-11T17:19:10.930633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518605022566347:2107] [node 5] ICS10 traffic start
2025-11-11T17:19:10.930660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:10.932643Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518605022566330:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:10.932899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518605022566317:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:10.932946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:10.932980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518605022566317:2106] [node 6] ICP22 created new session: [1:7571518605022566350:2106]
2025-11-11T17:19:10.933017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518605022566350:2106] [node 6] ICS09 handshake done sender: [1:7571518605022566330:2114] self: [1:7571518605022566324:0] peer: [6:7571518604211656284:0] socket: 230 qp: -1
2025-11-11T17:19:10.933040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518605022566350:2106] [node 6] ICS10 traffic start
2025-11-11T17:19:10.933073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.862178Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518606834925229:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.781352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518601046860563:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:10.902718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518606048125831:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.978398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518600727598957:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(112938299ns) {HandleHandshakeStatus:329(112935462ns) {DropIncomingHandshake:441(16900ns) DropOutgoingHandshake:456(215ns) SwitchToState:178(297ns) ProcessPendingSessionEvents:495(11673817ns) {ForwardSessionEventToSession:612(11620744ns) ForwardSessionEventToSession:612(5427ns) ForwardSessionEventToSession:612(3138ns) ForwardSessionEventToSession:612(4494ns) ForwardSessionEventToSession:612(6462ns) ForwardSessionEventToSession:612(2625ns) ForwardSessionEventToSession:612(11373ns) ForwardSessionEventToSession:612(2693ns) ForwardSessionEventToSession:612(4422ns) ForwardSessionEventToSession:612(1676ns)}}}
2025-11-11T17:19:10.862368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518606834925161:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.933873Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:10.943527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518606048125843:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.781384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.945814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518606048125844:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
2025-11-11T17:19:10.969794Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518606048125843:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.993957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518606048125845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
2025-11-11T17:19:10.998325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518606048125831:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.862442Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518606834925234:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:11.005808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518606048125844:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.012727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518606048125845:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.885977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518606834925162:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.012910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518601753158472:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.997752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518605226677792:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.013120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518601753158470:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:10.997785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518605226677792:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:11.011124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518605226677794:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.011143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518605226677794:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:11.012053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518609521645103:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17453
2025-11-11T17:19:11.012361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518605226677806:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17453
2025-11-11T17:19:10.781415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518601046860563:2048] [node 1] ICP22 created new session: [3:7571518605341827939:2048]
2025-11-11T17:19:10.781466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518605341827939:2048] [node 1] ICS09 handshake done sender: [3:7571518605341827934:2113] self: [3:7571518605341827937:0] peer: [1:7571518605022566325:0] socket: 238 qp: -1
2025-11-11T17:19:10.781496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518605341827939:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:10.781525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:10.842244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518605341827919:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(192393820ns) {Configure:104(192385541ns) {ProcessConfigured:135(47223ns) {StartInitialHandshake:157(45935ns) {DropHandshakes:469(2494ns) {DropIncomingHandshake:441(503ns) DropOutgoingHandshake:456(408ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:476(1561ns) {SwitchToState:178(433ns)}}}}}
2025-11-11T17:19:10.890760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518605341827944:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
2025-11-11T17:19:10.909895Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518605341827936:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.909918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518605341827936:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.910755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518605341827945:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
2025-11-11T17:19:10.921979Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518605341827936:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:10.922363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518605341827933:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:10.922423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:10.922471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518605341827933:2063] [node 6] ICP22 created new session: [3:7571518605341827947:2063]
2025-11-11T17:19:10.922510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518605341827947:2063] [node 6] ICS09 handshake done sender: [3:7571518605341827936:2114] self: [3:7571518605341827935:0] peer: [6:7571518604211656286:0] socket: 228 qp: -1
2025-11-11T17:19:10.922542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518605341827947:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:10.922597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:10.929809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518601046860563:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(112288246ns) {HandleHandshakeStatus:329(112287016ns) {DropIncomingHandshake:441(388ns) DropOutgoingHandshake:456(3445ns) SwitchToState:178(146ns) ProcessPendingSessionEvents:495(111424211ns) {ForwardSessionEventToSession:612(51447426ns) ForwardSessionEventToSession:612(8034533ns) ForwardSessionEventToSession:612(15181826ns) ForwardSessionEventToSession:612(10789ns) ForwardSessionEventToSession:612(8740498ns) ForwardSessionEventToSession:612(3227ns) ForwardSessionEventToSession:612(3899ns) ForwardSessionEventToSession:612(7506477ns) ForwardSessionEventToSession:612(2732ns) ForwardSessionEventToSession:612(4337571ns)}}}
2025-11-11T17:19:10.938081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518605341827950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
2025-11-11T17:19:10.965987Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518605341827950:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.966099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518605341827933:2063] [node 6] ICP18 handshake (actor [3:7571518605341827950:2119]) is too old
2025-11-11T17:19:10.989794Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518605341827945:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.001720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518601046860564:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:11.013937Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518605341827943:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.013959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518605341827943:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.886039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:10.886084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518606834925162:2049] [node 2] ICP22 created new session: [5:7571518606834925247:2049]
2025-11-11T17:19:10.886141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518606834925247:2049] [node 2] ICS09 handshake done sender: [5:7571518606834925234:2115] self: [5:7571518606834925240:0] peer: [2:7571518605226677782:0] socket: 247 qp: -1
2025-11-11T17:19:10.886174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518606834925247:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:10.886218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:10.887476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518606834925246:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:10.942563Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518606834925236:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.942606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518606834925236:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:10.943244Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518606834925233:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:10.959256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518606834925161:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:10.959305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:10.959337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518606834925161:2048] [node 1] ICP22 created new session: [5:7571518606834925252:2048]
2025-11-11T17:19:10.959369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518606834925252:2048] [node 1] ICS09 handshake done sender: [5:7571518606834925233:2114] self: [5:7571518606834925244:0] peer: [1:7571518605022566322:0] socket: 246 qp: -1
2025-11-11T17:19:10.959394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518606834925252:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:10.959442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:10.960120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518606834925250:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:10.960811Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518606834925236:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:10.960988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518606834925230:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:10.961038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:10.961077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518606834925230:2091] [node 6] ICP22 created new session: [5:7571518606834925254:2091]
2025-11-11T17:19:10.961112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518606834925254:2091] [node 6] ICS09 handshake done sender: [5:7571518606834925236:2116] self: [5:7571518606834925235:0] peer: [6:7571518604211656285:0] socket: 250 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.961137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518606834925254:2091] [node 6] ICS10 traffic start
2025-11-11T17:19:10.961166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:10.989662Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518606834925250:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:10.994050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518606834925163:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:11.025277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518610343093149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:11.038220Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518606048125831:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:11.038391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518601753158472:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:10.934891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518604211656203:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518604211656282:2116] held: no
2025-11-11T17:19:11.038438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:11.038476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518601753158472:2050] [node 3] ICP22 created new session: [4:7571518610343093151:2050]
2025-11-11T17:19:11.038504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518610343093151:2050] [node 3] ICS09 handshake done sender: [4:7571518606048125831:2110] self: [4:7571518610343093146:0] peer: [3:7571518605341827922:0] socket: 239 qp: -1
2025-11-11T17:19:11.038529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518610343093151:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:11.038570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:11.038924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518610343093150:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
2025-11-11T17:19:11.039411Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518606048125845:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:11.039578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518601753158470:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.039607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:11.039634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518601753158470:2048] [node 1] ICP22 created new session: [4:7571518610343093153:2048]
2025-11-11T17:19:11.039677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518610343093153:2048] [node 1] ICS09 handshake done sender: [4:7571518606048125845:2119] self: [4:7571518610343093148:0] peer: [1:7571518605022566332:0] socket: 271 qp: -1
2025-11-11T17:19:11.039714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518610343093153:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:11.039759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:11.055666Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518606048125841:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.055685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518606048125841:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:11.056551Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518606048125841:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:11.056719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518606048125830:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:11.056764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:11.056806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518606048125830:2071] [node 6] ICP22 created new session: [4:7571518610343093157:2071]
2025-11-11T17:19:11.056841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518610343093157:2071] [node 6] ICS09 handshake done sender: [4:7571518606048125841:2115] self: [4:7571518606048125837:0] peer: [6:7571518608506623597:0] socket: 277 qp: -1
2025-11-11T17:19:11.056867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518610343093157:2071] [node 6] ICS10 traffic start
2025-11-11T17:19:11.056903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:11.058009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518606048125832:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
2025-11-11T17:19:11.022011Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518605341827944:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.013434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518609317533648:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3089
2025-11-11T17:19:11.016158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518605226677792:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:10.937691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656289:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:11.017827Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518605226677794:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:11.033715Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518605022566334:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.089923Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518606048125832:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.022140Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518605341827923:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.101803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518601753158471:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:11.033743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518605022566334:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:11.034838Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518605022566334:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:11.035085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518605022566315:2105] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.937755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656290:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:11.070986Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518605226677806:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.022159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518605341827923:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:11.035123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:11.107326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518611129892554:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:11.035167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518605022566315:2105] [node 4] ICP22 created new session: [1:7571518609317533650:2105]
2025-11-11T17:19:11.035203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518609317533650:2105] [node 4] ICS09 handshake done sender: [1:7571518605022566334:2116] self: [1:7571518605022566332:0] peer: [4:7571518610343093148:0] socket: 265 qp: -1
2025-11-11T17:19:11.035236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518609317533650:2105] [node 4] ICS10 traffic start
2025-11-11T17:19:11.035267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:11.073635Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:11.073758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518605022566315:2105] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:10.946048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656287:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:10.986252Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518604211656282:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:10.990175Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518604211656283:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:11.002001Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:11.002930Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:11.003916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518604211656289:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.003955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518604211656205:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.004005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:11.004045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518604211656205:2050] [node 3] ICP22 created new session: [6:7571518608506623589:2050]
2025-11-11T17:19:11.004081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518608506623589:2050] [node 3] ICS09 handshake done sender: [6:7571518604211656283:2117] self: [6:7571518604211656286:0] peer: [3:7571518605341827935:0] socket: 260 qp: -1
2025-11-11T17:19:11.004108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518608506623589:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:11.004138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:11.004357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518604211656205:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:11.004597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518604211656203:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.004618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:11.004647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518604211656203:2048] [node 1] ICP22 created new session: [6:7571518608506623591:2048]
2025-11-11T17:19:11.004680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518608506623591:2048] [node 1] ICS09 handshake done sender: [6:7571518604211656282:2116] self: [6:7571518604211656284:0] peer: [1:7571518605022566324:0] socket: 255 qp: -1
2025-11-11T17:19:11.004703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518608506623591:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:11.004722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:11.004958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518604211656219:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518604211656281:2115] held: no
2025-11-11T17:19:11.005269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656291:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:11.005585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518604211656292:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:11.010046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518604211656204:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:11.010269Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518604211656281:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:11.025738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518604211656219:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.025824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:11.025861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518604211656219:2059] [node 5] ICP22 created new session: [6:7571518608506623595:2059]
2025-11-11T17:19:11.025901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518608506623595:2059] [node 5] ICS09 handshake done sender: [6:7571518604211656281:2115] self: [6:7571518604211656285:0] peer: [5:7571518606834925235:0] socket: 254 qp: -1
2025-11-11T17:19:11.025931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518608506623595:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:11.025959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:11.055337Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518604211656292:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.055447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518604211656225:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:11.069789Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:11.105628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518604211656225:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518604211656292:2122] held: no
2025-11-11T17:19:11.106937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518608506623598:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:11.134888Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518604211656289:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:11.135125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518604211656204:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.135173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:11.135212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518604211656204:2049] [node 2] ICP22 created new session: [6:7571518608506623600:2049]
2025-11-11T17:19:11.135252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518608506623600:2049] [node 2] ICS09 handshake done sender: [6:7571518604211656289:2119] self: [6:7571518608506623594:0] peer: [2:7571518605226677793:0] socket: 275 qp: -1
2025-11-11T17:19:11.135285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518608506623600:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:11.135314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:11.135877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518608506623599:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-11T17:19:11.115519Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:11.078559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518600931710419:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:11.022998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518609636795248:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:11.163221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518604211656225:2066] [node 4] ICP35 event processing took too much time PendingConnection:278(32252540ns) {HandleHandshakeStatus:395(32250485ns) {LogHandshakeFail:472(13645ns) {UpdateErrorStateLog:544(4361ns)} DropOutgoingHandshake:456(2019ns) ProcessPendingSessionEvents:495(16393483ns) {DropSessionEvent:512(9832ns) DropSessionEvent:512(5110ns) DropSessionEvent:512(4777ns) DropSessionEvent:512(3561ns) DropSessionEvent:512(4001ns) DropSessionEvent:512(2950ns) DropSessionEvent:512(18180ns) DropSessionEvent:512(3793ns) DropSessionEvent:512(7944528ns) DropSessionEvent:512(377424ns)}}}
2025-11-11T17:19:11.171054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518604211656292:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:11.121986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518606834925163:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518606834925250:2120] held: no
2025-11-11T17:19:11.078657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:11.023472Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518605341827945:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:11.037756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518601046860564:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.170050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518610343093160:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5392
2025-11-11T17:19:11.037810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:11.188210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518604211656225:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.078703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518600931710419:2049] [node 3] ICP22 created new session: [2:7571518609521645106:2049]
2025-11-11T17:19:11.188293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:11.188341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518604211656225:2066] [node 4] ICP22 created new session: [6:7571518608506623602:2066]
2025-11-11T17:19:11.188379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518608506623602:2066] [node 4] ICS09 handshake done sender: [6:7571518604211656292:2122] self: [6:7571518608506623597:0] peer: [4:7571518606048125837:0] socket: 280 qp: -1
2025-11-11T17:19:11.188403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518608506623602:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:11.188431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:11.133644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518611129892553:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:11.187014Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518606048125832:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:11.078755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518609521645106:2049] [node 3] ICS09 handshake done sender: [2:7571518605226677792:2115] self: [2:7571518605226677791:0] peer: [3:7571518605341827951:0] socket: 241 qp: -1
2025-11-11T17:19:11.037858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518601046860564:2049] [node 2] ICP22 created new session: [3:7571518609636795251:2049]
2025-11-11T17:19:11.209982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518601753158471:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.210038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:11.210087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518601753158471:2049] [node 2] ICP22 created new session: [4:7571518610343093161:2049]
2025-11-11T17:19:11.210126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518610343093161:2049] [node 2] ICS09 handshake done sender: [4:7571518606048125832:2111] self: [4:7571518610343093159:0] peer: [2:7571518605226677784:0] socket: 245 qp: -1
2025-11-11T17:19:11.210162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518610343093161:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:11.210195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:11.210781Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11588) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:11.216440Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518606048125836:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.216479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518606048125836:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:11.216975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518601753158471:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:11.037905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518609636795251:2049] [node 2] ICS09 handshake done sender: [3:7571518605341827945:2118] self: [3:7571518605341827951:0] peer: [2:7571518605226677791:0] socket: 248 qp: -1
2025-11-11T17:19:11.037937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518609636795251:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:11.037969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:11.041767Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518605341827923:2106] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:11.091471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518601046860564:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(53548238ns) {HandleHandshakeStatus:329(53545133ns) {DropIncomingHandshake:441(771ns) DropOutgoingHandshake:456(5774ns) SwitchToState:178(400ns) ProcessPendingSessionEvents:495(9565687ns) {ForwardSessionEventToSession:612(9517307ns) ForwardSessionEventToSession:612(7582ns) ForwardSessionEventToSession:612(5542ns) ForwardSessionEventToSession:612(4765ns) ForwardSessionEventToSession:612(2785ns) ForwardSessionEventToSession:612(2833ns) ForwardSessionEventToSession:612(5131ns) ForwardSessionEventToSession:612(3213ns) ForwardSessionEventToSession:612(2923ns) ForwardSessionEventToSession:612(4983ns)}}}
2025-11-11T17:19:11.091555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518605341827875:2058] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:11.091593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:11.091634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518605341827875:2058] [node 4] ICP22 created new session: [3:7571518609636795253:2058]
2025-11-11T17:19:11.091674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518609636795253:2058] [node 4] ICS09 handshake done sender: [3:7571518605341827923:2106] self: [3:7571518605341827922:0] peer: [4:7571518610343093146:0] socket: 231 qp: -1
2025-11-11T17:19:11.091714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518609636795253:2058] [node 4] ICS10 traffic start
2025-11-11T17:19:11.091748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:11.113916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518605341827943:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:11.114291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518605341827919:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:11.114341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:11.114385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518605341827919:2065] [node 5] ICP22 created new session: [3:7571518609636795256:2065]
2025-11-11T17:19:11.114422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518609636795256:2065] [node 5] ICS09 handshake done sender: [3:7571518605341827943:2116] self: [3:7571518605341827942:0] peer: [5:7571518606834925256:0] socket: 276 qp: -1
2025-11-11T17:19:11.114453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518609636795256:2065] [node 5] ICS10 traffic start
2025-11-11T17:19:11.114479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:11.078787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518609521645106:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:11.078846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:11.079203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518605226677774:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:11.079271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:11.079313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518605226677774:2077] [node 6] ICP22 created new session: [2:7571518609521645108:2077]
2025-11-11T17:19:11.079350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518609521645108:2077] [node 6] ICS09 handshake done sender: [2:7571518605226677794:2116] self: [2:7571518605226677793:0] peer: [6:7571518608506623594:0] socket: 274 qp: -1
2025-11-11T17:19:11.079378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518609521645108:2077] [node 6] ICS10 traffic start
2025-11-11T17:19:11.079403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:11.133704Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518605226677785:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.133749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518605226677785:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:11.134339Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518609521645103:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.134539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518605226677774:2077] [node 6] ICP18 handshake (actor [2:7571518609521645103:2120]) is too old
2025-11-11T17:19:11.162603Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518605226677785:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:11.187204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518605226677773:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:11.187279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:11.187321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518605226677773:2075] [node 4] ICP22 created new session: [2:7571518609521645112:2075]
2025-11-11T17:19:11.187371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518609521645112:2075] [node 4] ICS09 handshake done sender: [2:7571518605226677785:2112] self: [2:7571518605226677784:0] peer: [4:7571518610343093159:0] socket: 243 qp: -1
2025-11-11T17:19:11.187398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518609521645112:2075] [node 4] ICS10 traffic start
2025-11-11T17:19:11.187441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:11.161894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518606834925163:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(21078117ns) {HandleHandshakeStatus:395(21076630ns) {LogHandshakeFail:472(12138ns) {UpdateErrorStateLog:544(4467ns)} DropOutgoingHandshake:456(1539ns) ProcessPendingSessionEvents:495(15212226ns) {DropSessionEvent:512(9536ns) DropSessionEvent:512(4592ns) DropSessionEvent:512(6509ns) DropSessionEvent:512(26475ns) DropSessionEvent:512(7314402ns) DropSessionEvent:512(5723ns) DropSessionEvent:512(5318ns) DropSessionEvent:512(5073ns) DropSessionEvent:512(5624ns) DropSessionEvent:512(3441ns)}}}
2025-11-11T17:19:11.162417Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518606834925250:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:11.170610Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518611129892553:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:11.178821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518606834925195:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:11.185997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518606834925163:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.186057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:11.186103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518606834925163:2050] [node 3] ICP22 created new session: [5:7571518611129892556:2050]
2025-11-11T17:19:11.186152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518611129892556:2050] [node 3] ICS09 handshake done sender: [5:7571518606834925250:2120] self: [5:7571518606834925256:0] peer: [3:7571518605341827942:0] socket: 278 qp: -1
2025-11-11T17:19:11.186187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518611129892556:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:11.186240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:11.218685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518611129892558:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27172
2025-11-11T17:19:11.220067Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518606048125836:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:11.226418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518606048125791:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:11.226502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:11.226546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518606048125791:2062] [node 5] ICP22 created new session: [4:7571518610343093164:2062]
2025-11-11T17:19:11.226586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518610343093164:2062] [node 5] ICS09 handshake done sender: [4:7571518606048125836:2113] self: [4:7571518606048125835:0] peer: [5:7571518611129892555:0] socket: 292 qp: -1
2025-11-11T17:19:11.226618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518610343093164:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:11.226651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:11.227784Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518611129892553:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:11.228079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518606834925195:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:11.228141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:11.228181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518606834925195:2064] [node 4] ICP22 created new session: [5:7571518611129892559:2064]
2025-11-11T17:19:11.228216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518611129892559:2064] [node 4] ICS09 handshake done sender: [5:7571518611129892553:2121] self: [5:7571518611129892555:0] peer: [4:7571518606048125835:0] socket: 293 qp: -1
2025-11-11T17:19:11.228243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518611129892559:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:11.228270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:11.285488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:11.284999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:11.285247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518605341827939:2048] [node 1] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-11T17:19:11.285279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518605341827939:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.285353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:11.285386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518605341827939:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:11.286923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518605226677803:2048] [node 1] ICS01 socket: 240 reason# EndOfStream
2025-11-11T17:19:11.286973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518600931710418:2048] [node 1] ICP30 unregister session Session# [2:7571518605226677803:2048] VirtualId# [2:7571518605226677795:0]
2025-11-11T17:19:11.287011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518605226677803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.287082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:11.287122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:11.293306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:11.293462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518608506623591:2048] [node 1] ICS01 socket: 255 reason# EndOfStream
2025-11-11T17:19:11.292943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:11.293488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518604211656203:2048] [node 1] ICP30 unregister session Session# [6:7571518608506623591:2048] VirtualId# [6:7571518604211656284:0]
2025-11-11T17:19:11.293101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518610343093153:2048] [node 1] ICS01 socket: 271 reason# EndOfStream
2025-11-11T17:19:11.293506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518608506623591:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.293131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518601753158470:2048] [node 1] ICP30 unregister session Session# [4:7571518610343093153:2048] VirtualId# [4:7571518610343093148:0]
2025-11-11T17:19:11.293551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:11.293150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518610343093153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.293581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:11.293194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:11.292456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:11.293229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:11.292668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518606834925252:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-11-11T17:19:11.292704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518606834925161:2048] [node 1] ICP30 unregister session Session# [5:7571518606834925252:2048] VirtualId# [5:7571518606834925244:0]
2025-11-11T17:19:11.292732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518606834925252:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.292808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:11.292862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:11.294550Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:19:11.295196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518601046860563:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:19:11.295234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518605341827939:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:11.295856Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Couldn't connect to any resolved address
2025-11-11T17:19:11.296068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518601046860563:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:11.296108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518605341827939:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:11.296133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518601046860563:2048] [node 1] ICP30 unregister session Session# [3:7571518605341827939:2048] VirtualId# [3:7571518605341827937:0]
2025-11-11T17:19:11.296168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:11.296196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518601046860563:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:11.296209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4364) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:11.304127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518601046860563:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:11.353735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:11.353968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518609636795251:2049] [node 2] ICS01 socket: 248 reason# EndOfStream
2025-11-11T17:19:11.354007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518601046860564:2049] [node 2] ICP30 unregister session Session# [3:7571518609636795251:2049] VirtualId# [3:7571518605341827951:0]
2025-11-11T17:19:11.354038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518609636795251:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.354124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:11.354166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:11.361782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:11.361982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518606834925247:2049] [node 2] ICS01 socket: 247 reason# EndOfStream
2025-11-11T17:19:11.362024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518606834925162:2049] [node 2] ICP30 unregister session Session# [5:7571518606834925247:2049] VirtualId# [5:7571518606834925240:0]
2025-11-11T17:19:11.362056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518606834925247:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.362128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:11.362169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:11.362245Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:11.362399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518608506623600:2049] [node 2] ICS01 socket: 275 reason# EndOfStream
2025-11-11T17:19:11.362433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518604211656204:2049] [node 2] ICP30 unregister session Session# [6:7571518608506623600:2049] VirtualId# [6:7571518608506623594:0]
2025-11-11T17:19:11.362455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518608506623600:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.362496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:11.362524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:11.365733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:11.365946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518610343093161:2049] [node 2] ICS01 socket: 245 reason# EndOfStream
2025-11-11T17:19:11.365987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518601753158471:2049] [node 2] ICP30 unregister session Session# [4:7571518610343093161:2049] VirtualId# [4:7571518610343093159:0]
2025-11-11T17:19:11.366018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518610343093161:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.366092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:11.366128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:11.505228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:11.504380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:11.504599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518611129892556:2050] [node 3] ICS01 socket: 278 reason# EndOfStream
2025-11-11T17:19:11.504639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518606834925163:2050] [node 3] ICP30 unregister session Session# [5:7571518611129892556:2050] VirtualId# [5:7571518606834925256:0]
2025-11-11T17:19:11.504670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518611129892556:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.504732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:11.504767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:11.504850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:11.505012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518610343093151:2050] [node 3] ICS01 socket: 239 reason# EndOfStream
2025-11-11T17:19:11.505043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518601753158472:2050] [node 3] ICP30 unregister session Session# [4:7571518610343093151:2050] VirtualId# [4:7571518610343093146:0]
2025-11-11T17:19:11.505062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518610343093151:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.505118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:11.505151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:11.505396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518608506623589:2050] [node 3] ICS01 socket: 260 reason# EndOfStream
2025-11-11T17:19:11.505424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518604211656205:2050] [node 3] ICP30 unregister session Session# [6:7571518608506623589:2050] VirtualId# [6:7571518604211656286:0]
2025-11-11T17:19:11.505441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518608506623589:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.505487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:11.505518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:11.585749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:11.585972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518611129892559:2064] [node 4] ICS01 socket: 293 reason# EndOfStream
2025-11-11T17:19:11.586005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518606834925195:2064] [node 4] ICP30 unregister session Session# [5:7571518611129892559:2064] VirtualId# [5:7571518611129892555:0]
2025-11-11T17:19:11.586034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518611129892559:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.586112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:11.586153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:11.585759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:11.585973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518608506623602:2066] [node 4] ICS01 socket: 280 reason# EndOfStream
2025-11-11T17:19:11.586008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518604211656225:2066] [node 4] ICP30 unregister session Session# [6:7571518608506623602:2066] VirtualId# [6:7571518608506623597:0]
2025-11-11T17:19:11.586034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518608506623602:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.586116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:11.586152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:11.686064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:11.686316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518608506623595:2059] [node 5] ICS01 socket: 254 reason# EndOfStream
2025-11-11T17:19:11.686357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518604211656219:2059] [node 5] ICP30 unregister session Session# [6:7571518608506623595:2059] VirtualId# [6:7571518604211656285:0]
2025-11-11T17:19:11.686389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518608506623595:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:11.686489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:11.686525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:11.934117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518611785326202:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:11.934160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518611785326203:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:11.989887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518611385648614:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:11.989931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518611385648615:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:12.048633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518614709856903:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:12.048681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518614709856904:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:12.122034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518616564980883:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:12.122074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518616564980884:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:12.122099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518616564980885:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:12.105937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518612669868413:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:12.105977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518612669868414:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:12.106002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518612669868415:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:12.281267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518613208070808:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:12.281304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518613208070809:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:12.281329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518613208070810:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:12.413229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518616564980900:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:12.411429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518612669868431:2064] [node 6] ICP01 ready to work
2025-11-11T17:19:12.473670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518614709856923:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.443674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518616080293535:2080] [node 6] ICP01 ready to work
2025-11-11T17:19:12.450617Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518612669868413:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25774907ns) {Configure:104(25770649ns) {ProcessConfigured:135(65359ns) {StartInitialHandshake:157(62888ns) {DropHandshakes:469(2044ns) {DropIncomingHandshake:441(476ns) DropOutgoingHandshake:456(299ns)} GenerateSessionVirtualId:988(1312ns) PrepareNewSessionHandshake:476(1163ns) {SwitchToState:178(433ns)}}}}}
2025-11-11T17:19:12.558038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518616564980916:2069] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.505349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518615680615973:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62053
2025-11-11T17:19:12.533797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518616080293561:2077] [node 4] ICP01 ready to work
2025-11-11T17:19:12.494883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518612669868465:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:12.580291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518614709856967:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:12.595599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518614709856904:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(81708141ns) {Configure:104(81701734ns) {ProcessConfigured:135(54566ns) {StartInitialHandshake:157(51938ns) {DropHandshakes:469(2524ns) {DropIncomingHandshake:441(718ns) DropOutgoingHandshake:456(552ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:476(1455ns) {SwitchToState:178(636ns)}}}}}
2025-11-11T17:19:12.553770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518615680615980:2076] [node 6] ICP01 ready to work
2025-11-11T17:19:12.621351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518616080293563:2082] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.650813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518614709856972:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17732
2025-11-11T17:19:12.651019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518614709856973:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17732
2025-11-11T17:19:12.651746Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518614709856972:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.613883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518615680615982:2073] [node 4] ICP01 ready to work
2025-11-11T17:19:12.660455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518613208070843:2068] [node 4] ICP01 ready to work
2025-11-11T17:19:12.573101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518612669868479:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
2025-11-11T17:19:12.666751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518614709856976:2063] [node 4] ICP01 ready to work
2025-11-11T17:19:12.644683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518615680615981:2081] [node 5] ICP01 ready to work
2025-11-11T17:19:12.658887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518613208070851:2061] [node 5] ICP01 ready to work
2025-11-11T17:19:12.653785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518615680615973:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.704393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518614709856923:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(81234237ns) {Configure:104(81226357ns) {ProcessConfigured:135(61344ns) {StartInitialHandshake:157(59024ns) {DropHandshakes:469(2292ns) {DropIncomingHandshake:441(489ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(739ns) PrepareNewSessionHandshake:476(1784ns) {SwitchToState:178(637ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.744270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518615680615983:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62053
2025-11-11T17:19:12.727606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518611785326203:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(207485314ns) {Configure:104(207478581ns) {ProcessConfigured:135(66493ns) {StartInitialHandshake:157(59616ns) {DropHandshakes:469(2309ns) {DropIncomingHandshake:441(486ns) DropOutgoingHandshake:456(462ns)} GenerateSessionVirtualId:988(919ns) PrepareNewSessionHandshake:476(1411ns) {SwitchToState:178(518ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.783140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518616564980883:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(285193748ns) {Configure:104(285189127ns) {ProcessConfigured:135(61158ns) {StartInitialHandshake:157(58722ns) {DropHandshakes:469(2924ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(754ns)} GenerateSessionVirtualId:988(743ns) PrepareNewSessionHandshake:476(1504ns) {SwitchToState:178(761ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.794010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518614709856923:2064] [node 5] ICP35 event processing took too much time PendingConnection:278(24896375ns) {IncomingHandshake:261(45299ns) {DropIncomingHandshake:441(445ns) PrepareNewSessionHandshake:476(2054ns) {SwitchToState:178(966ns)} GenerateSessionVirtualId:988(580ns) IssueIncomingHandshakeReply:193(6125ns)}}
2025-11-11T17:19:12.794456Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518614709856973:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.794840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518614709856980:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17732
2025-11-11T17:19:12.794956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518614709856904:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:12.795271Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518615680615977:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.795308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518615680615977:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.796718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518616564980959:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:12.741886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518612669868479:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.803808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518614709856976:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(105751072ns) {Configure:104(105746106ns) {ProcessConfigured:135(51693ns) {StartInitialHandshake:157(49075ns) {DropHandshakes:469(4729ns) {DropIncomingHandshake:441(484ns) DropOutgoingHandshake:456(462ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:476(1182ns) {SwitchToState:178(468ns)}}}}}
2025-11-11T17:19:12.796679Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518615680615977:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:12.818295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518616564980884:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(404808513ns) {Configure:104(404805108ns) {ProcessConfigured:135(11958662ns) {StartInitialHandshake:157(11956054ns) {DropHandshakes:469(1332ns) {DropIncomingHandshake:441(311ns) DropOutgoingHandshake:456(293ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:476(3891ns) {SwitchToState:178(2123ns)}}}}}
2025-11-11T17:19:12.729428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518616080293564:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34063
2025-11-11T17:19:12.842426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518614709856967:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(245412113ns) {Configure:104(245409003ns) {ProcessConfigured:135(74944ns) {StartInitialHandshake:157(70455ns) {DropHandshakes:469(2517ns) {DropIncomingHandshake:441(509ns) DropOutgoingHandshake:456(470ns)} GenerateSessionVirtualId:988(995ns) PrepareNewSessionHandshake:476(1876ns) {SwitchToState:178(993ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.822480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518615680615982:2073] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(140312506ns) {Configure:104(140307989ns) {ProcessConfigured:135(51427ns) {StartInitialHandshake:157(48718ns) {DropHandshakes:469(4423ns) {DropIncomingHandshake:441(369ns) DropOutgoingHandshake:456(285ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:476(991ns) {SwitchToState:178(363ns)}}}}}
2025-11-11T17:19:12.729582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518616080293565:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34063
2025-11-11T17:19:12.768845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518612669868485:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.886056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518613208070843:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(212095515ns) {Configure:104(212090134ns) {ProcessConfigured:135(70177ns) {StartInitialHandshake:157(67265ns) {DropHandshakes:469(2032ns) {DropIncomingHandshake:441(420ns) DropOutgoingHandshake:456(519ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:476(1157ns) {SwitchToState:178(428ns)}}}}}
2025-11-11T17:19:12.793964Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518612669868485:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.866699Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518614709856980:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.765965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518616080293566:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34063
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.897806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518613208070851:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(195969011ns) {Configure:104(195959989ns) {ProcessConfigured:135(67787ns) {StartInitialHandshake:157(65739ns) {DropHandshakes:469(17217ns) {DropIncomingHandshake:441(513ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:476(987ns) {SwitchToState:178(665ns)}}}}}
2025-11-11T17:19:12.881757Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518616564980959:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.830470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518615680615980:2076] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(178117714ns) {Configure:104(178113335ns) {ProcessConfigured:135(65645ns) {StartInitialHandshake:157(63367ns) {DropHandshakes:469(2073ns) {DropIncomingHandshake:441(500ns) DropOutgoingHandshake:456(399ns)} GenerateSessionVirtualId:988(1078ns) PrepareNewSessionHandshake:476(1033ns) {SwitchToState:178(507ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.897910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518613208070861:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:12.794106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518612669868413:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:12.766060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518616080293567:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34063
2025-11-11T17:19:12.881134Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518615680615983:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.881783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518614709856903:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.895716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518616564980885:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:12.898228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518613208070879:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:12.894917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518614709856983:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17732
2025-11-11T17:19:12.935513Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518614709856978:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.935549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518614709856978:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:12.805843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518612669868431:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(394091032ns) {Configure:104(394085824ns) {ProcessConfigured:135(11898591ns) {StartInitialHandshake:157(11894721ns) {DropHandshakes:469(2889ns) {DropIncomingHandshake:441(561ns) DropOutgoingHandshake:456(554ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:476(4055ns) {SwitchToState:178(2407ns)}}}}}
2025-11-11T17:19:12.933838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518613208070810:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(399683002ns) {Configure:104(399677923ns) {ProcessConfigured:135(16441186ns) {StartInitialHandshake:157(16438672ns) {DropHandshakes:469(2384ns) {DropIncomingHandshake:441(575ns) DropOutgoingHandshake:456(457ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:476(2775ns) {SwitchToState:178(1112ns)}}}}}
2025-11-11T17:19:12.951725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518613208070879:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.951868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518613208070843:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:12.953734Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518613208070861:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.960771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518613208070887:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:12.965958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518613208070851:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.766337Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293564:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.901316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518611385648615:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:12.990174Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518614709856978:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.781864Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293565:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.971241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518616564980902:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.901422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:12.985336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518613208070809:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(539382499ns) {Configure:104(539377523ns) {ProcessConfigured:135(12580024ns) {StartInitialHandshake:157(12577830ns) {DropHandshakes:469(1823ns) {DropIncomingHandshake:441(406ns) DropOutgoingHandshake:456(352ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:476(2678ns) {SwitchToState:178(997ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.846239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518612669868490:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.971278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518616564980902:2064] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.002790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518613208070892:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:13.003085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518613208070891:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:13.005999Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518613208070887:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.010330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518617503038189:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:13.014044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518613208070809:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.998139Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518614709856973:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:12.998483Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:13.004201Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:12.823163Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293571:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.901472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518611385648615:2049] [node 3] ICP22 created new session: [2:7571518615680615992:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.006578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518614709856903:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518614709856980:2115] held: no
2025-11-11T17:19:12.823197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518616080293571:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:12.854750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518612669868491:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.979204Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518616564980902:2064] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:13.075964Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518613208070891:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.076064Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24770) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:12.855418Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518612669868485:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:12.855558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518612669868413:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:12.855604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:12.855645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518612669868413:2048] [node 1] ICP22 created new session: [4:7571518612669868492:2048]
2025-11-11T17:19:12.855683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518612669868492:2048] [node 1] ICS09 handshake done sender: [4:7571518612669868485:2113] self: [4:7571518612669868487:0] peer: [1:7571518616080293570:0] socket: 256 qp: -1
2025-11-11T17:19:12.855722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518612669868492:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:12.855759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:12.888246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518612669868415:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(475862254ns) {Configure:104(475849025ns) {ProcessConfigured:135(8911212ns) {StartInitialHandshake:157(8908293ns) {DropHandshakes:469(1936ns) {DropIncomingHandshake:441(559ns) DropOutgoingHandshake:456(311ns)} GenerateSessionVirtualId:988(981ns) PrepareNewSessionHandshake:476(2703ns) {SwitchToState:178(929ns)}}}}}
2025-11-11T17:19:12.929768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518612669868465:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(372770993ns) {Configure:104(372766698ns) {ProcessConfigured:135(26407412ns) {StartInitialHandshake:157(26405371ns) {DropHandshakes:469(2405ns) {DropIncomingHandshake:441(590ns) DropOutgoingHandshake:456(373ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:476(2770ns) {SwitchToState:178(995ns)}}}}}
2025-11-11T17:19:12.930650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518612669868499:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
2025-11-11T17:19:12.941978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518612669868499:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.961063Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518612669868490:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.989506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518612669868414:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:12.997411Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518612669868488:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.997445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518612669868488:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.017662Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518612669868488:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:13.018027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518616964835800:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
2025-11-11T17:19:13.019307Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518612669868490:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:13.029909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518612669868431:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.029972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:13.038888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518612669868431:2064] [node 6] ICP22 created new session: [4:7571518616964835802:2064]
2025-11-11T17:19:13.041897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518612669868414:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.041941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:13.041984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518612669868414:2049] [node 2] ICP22 created new session: [4:7571518616964835803:2049]
2025-11-11T17:19:13.042019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518616964835803:2049] [node 2] ICS09 handshake done sender: [4:7571518612669868490:2115] self: [4:7571518612669868502:0] peer: [2:7571518615680615985:0] socket: 263 qp: -1
2025-11-11T17:19:13.042050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518616964835803:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:13.042082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:13.061914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518616964835802:2064] [node 6] ICS09 handshake done sender: [4:7571518612669868488:2114] self: [4:7571518612669868486:0] peer: [6:7571518613208070886:0] socket: 264 qp: -1
2025-11-11T17:19:13.068561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518616964835805:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
2025-11-11T17:19:13.068800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518616964835802:2064] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.985837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518616564980965:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:12.901514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518615680615992:2049] [node 3] ICS09 handshake done sender: [2:7571518615680615977:2112] self: [2:7571518615680615976:0] peer: [3:7571518614709856982:0] socket: 241 qp: -1
2025-11-11T17:19:13.076564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518613208070843:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24770) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518613208070879:2113] held: no
2025-11-11T17:19:13.094026Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518613208070861:2096] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:13.094249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518613208070808:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:13.094295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518613208070851:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.094350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:13.094385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518613208070851:2061] [node 5] ICP22 created new session: [6:7571518617503038194:2061]
2025-11-11T17:19:13.094424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518617503038194:2061] [node 5] ICS09 handshake done sender: [6:7571518613208070861:2096] self: [6:7571518613208070888:0] peer: [5:7571518616564980901:0] socket: 253 qp: -1
2025-11-11T17:19:13.094451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518617503038194:2061] [node 5] ICS10 traffic start
2025-11-11T17:19:13.094477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.098416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518617503038192:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:13.108889Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518616964835805:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.109019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518612669868415:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:12.853963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293567:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.119521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518616964835808:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49760
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.006672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518614709856904:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:12.901550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518615680615992:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:12.854919Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518616080293571:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:13.038561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518620859948262:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:13.074473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518616564980900:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.074509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:13.074553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518616564980900:2063] [node 6] ICP22 created new session: [5:7571518620859948264:2063]
2025-11-11T17:19:13.074595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518620859948264:2063] [node 6] ICS09 handshake done sender: [5:7571518616564980902:2064] self: [5:7571518616564980901:0] peer: [6:7571518613208070888:0] socket: 234 qp: -1
2025-11-11T17:19:13.074623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518620859948264:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:13.074654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:13.075214Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518616564980959:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:13.076643Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17477) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:13.091152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518620859948263:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:13.091328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518616564980885:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17477) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518616564980959:2114] held: no
2025-11-11T17:19:13.105801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518620859948262:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.105894Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518620859948263:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.105928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518616564980885:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.105972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:13.121286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518616564980916:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:13.121354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518616564980884:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:13.125782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518616564980964:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:13.126194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518620859948268:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:13.126328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518616564980885:2050] [node 3] ICP22 created new session: [5:7571518620859948267:2050]
2025-11-11T17:19:13.142034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518620859948267:2050] [node 3] ICS09 handshake done sender: [5:7571518616564980959:2114] self: [5:7571518616564980962:0] peer: [3:7571518614709856977:0] socket: 261 qp: -1
2025-11-11T17:19:12.901622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:13.006713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.154091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518617503038191:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:13.129789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.857039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518616080293561:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.164261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518613208070843:2068] [node 4] ICP35 event processing took too much time PendingConnection:278(21158977ns) {HandleHandshakeStatus:395(21156936ns) {LogHandshakeFail:472(11928ns) {UpdateErrorStateLog:544(3751ns)} DropOutgoingHandshake:456(278ns) ProcessPendingSessionEvents:495(21083603ns) {DropSessionEvent:512(7332ns) DropSessionEvent:512(3663ns) DropSessionEvent:512(23774ns) DropSessionEvent:512(4488ns) DropSessionEvent:512(5104ns) DropSessionEvent:512(5024ns) DropSessionEvent:512(3807ns) DropSessionEvent:512(4073ns) DropSessionEvent:512(3500ns) DropSessionEvent:512(4095ns)}}}
2025-11-11T17:19:13.169919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518613208070851:2061] [node 5] ICP35 event processing took too much time PendingConnection:278(47729210ns) {HandleHandshakeStatus:329(47727320ns) {DropIncomingHandshake:441(241ns) DropOutgoingHandshake:456(4041ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(47428570ns) {ForwardSessionEventToSession:612(16174761ns) ForwardSessionEventToSession:612(6831ns) ForwardSessionEventToSession:612(18905639ns) ForwardSessionEventToSession:612(12236ns) ForwardSessionEventToSession:612(2262317ns) ForwardSessionEventToSession:612(18480ns) ForwardSessionEventToSession:612(4169ns) ForwardSessionEventToSession:612(4262578ns) ForwardSessionEventToSession:612(3767ns) ForwardSessionEventToSession:612(3382ns)}}}
2025-11-11T17:19:13.170153Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:13.149981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518620859948267:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:13.006751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518614709856904:2049] [node 2] ICP22 created new session: [3:7571518619004824288:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.138364Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518616964835805:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:13.193692Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518613208070887:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:13.193937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518613208070879:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.902272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518611385648615:2049] [node 3] ICP18 handshake (actor [2:7571518615680615983:2113]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:12.857090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:13.006792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518619004824288:2049] [node 2] ICS09 handshake done sender: [3:7571518614709856973:2113] self: [3:7571518614709856982:0] peer: [2:7571518615680615976:0] socket: 245 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.138556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518612669868415:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.150046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:12.902440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518615680615991:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62053
2025-11-11T17:19:12.857131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518616080293561:2077] [node 4] ICP22 created new session: [1:7571518616080293577:2077]
2025-11-11T17:19:12.857173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518616080293577:2077] [node 4] ICS09 handshake done sender: [1:7571518616080293571:2115] self: [1:7571518616080293570:0] peer: [4:7571518612669868487:0] socket: 255 qp: -1
2025-11-11T17:19:12.857197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518616080293577:2077] [node 4] ICS10 traffic start
2025-11-11T17:19:12.857248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:12.870302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518611785326202:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(468383474ns) {Configure:104(468376832ns) {ProcessConfigured:135(56716ns) {StartInitialHandshake:157(54672ns) {DropHandshakes:469(2009ns) {DropIncomingHandshake:441(339ns) DropOutgoingHandshake:456(407ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:476(1870ns) {SwitchToState:178(617ns)}}}}}
2025-11-11T17:19:12.917816Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293569:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.917845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518616080293569:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:12.919285Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518616080293569:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:12.919528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518611785326203:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:12.919580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:12.919623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518611785326203:2049] [node 3] ICP22 created new session: [1:7571518616080293585:2049]
2025-11-11T17:19:12.919664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518616080293585:2049] [node 3] ICS09 handshake done sender: [1:7571518616080293569:2114] self: [1:7571518616080293568:0] peer: [3:7571518614709856988:0] socket: 254 qp: -1
2025-11-11T17:19:12.919693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518616080293585:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:12.919779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:12.953998Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293566:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.970671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518616080293535:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(392816655ns) {Configure:104(392809297ns) {ProcessConfigured:135(41373116ns) {StartInitialHandshake:157(32865004ns) {DropHandshakes:469(2818ns) {DropIncomingHandshake:441(491ns) DropOutgoingHandshake:456(455ns)} GenerateSessionVirtualId:988(1813ns) PrepareNewSessionHandshake:476(3439ns) {SwitchToState:178(1184ns)}}}}}
2025-11-11T17:19:12.981507Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293580:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.981552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518616080293580:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:12.997320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518616080293563:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(303126114ns) {Configure:104(303114535ns) {ProcessConfigured:135(45132728ns) {StartInitialHandshake:157(45129912ns) {DropHandshakes:469(1282ns) {DropIncomingHandshake:441(412ns) DropOutgoingHandshake:456(177ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:476(3580ns) {SwitchToState:178(1234ns)}}}}}
2025-11-11T17:19:13.029462Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518616080293580:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:13.037924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518611785326202:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.039073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:13.039111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518611785326202:2048] [node 2] ICP22 created new session: [1:7571518620375260889:2048]
2025-11-11T17:19:13.068745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518620375260889:2048] [node 2] ICS09 handshake done sender: [1:7571518616080293580:2116] self: [1:7571518616080293579:0] peer: [2:7571518615680615997:0] socket: 235 qp: -1
2025-11-11T17:19:13.085653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518620375260889:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:13.085713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:13.114369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518611785326202:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(84345952ns) {HandleHandshakeStatus:329(76255362ns) {DropIncomingHandshake:441(18216ns) DropOutgoingHandshake:456(1339ns) SwitchToState:178(508ns) ProcessPendingSessionEvents:495(96142ns) {ForwardSessionEventToSession:612(50735ns) ForwardSessionEventToSession:612(2753ns) ForwardSessionEventToSession:612(7472ns) ForwardSessionEventToSession:612(5589ns) ForwardSessionEventToSession:612(2381ns) ForwardSessionEventToSession:612(7789ns) ForwardSessionEventToSession:612(2839ns) ForwardSessionEventToSession:612(4631ns) ForwardSessionEventToSession:612(2830ns) ForwardSessionEventToSession:612(2156ns)}}}
2025-11-11T17:19:13.149703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293589:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.149743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518616080293589:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.154371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518620375260892:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34063
2025-11-11T17:19:13.162974Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518616080293589:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:13.163186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518616080293535:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.163268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:13.163310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518616080293535:2080] [node 6] ICP22 created new session: [1:7571518620375260894:2080]
2025-11-11T17:19:13.163353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518620375260894:2080] [node 6] ICS09 handshake done sender: [1:7571518616080293589:2118] self: [1:7571518616080293587:0] peer: [6:7571518617503038193:0] socket: 275 qp: -1
2025-11-11T17:19:13.163379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518620375260894:2080] [node 6] ICS10 traffic start
2025-11-11T17:19:13.163405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:13.169759Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518620375260892:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.197920Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518616080293588:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.197945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518616080293588:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.201568Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518616080293588:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:13.211668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518616080293563:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.211737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.154047Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:12.944859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518615680615991:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:12.945108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518611385648614:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:12.977736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518615680615981:2081] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(260116166ns) {Configure:104(260111097ns) {ProcessConfigured:135(59286ns) {StartInitialHandshake:157(56825ns) {DropHandshakes:469(2749ns) {DropIncomingHandshake:441(425ns) DropOutgoingHandshake:456(621ns)} GenerateSessionVirtualId:988(1900ns) PrepareNewSessionHandshake:476(1748ns) {SwitchToState:178(509ns)}}}}}
2025-11-11T17:19:13.001633Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518615680615986:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.001679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518615680615986:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.201976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518613208070808:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518613208070891:2118] held: no
2025-11-11T17:19:13.006823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518619004824288:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:13.138619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:13.242146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518616080293563:2082] [node 5] ICP22 created new session: [1:7571518620375260898:2082]
2025-11-11T17:19:13.010533Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518615680615986:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:13.037733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518615680615990:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.037772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518615680615990:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.049933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518619975583296:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62053
2025-11-11T17:19:13.057733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518615680615982:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.068864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:13.068913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518615680615982:2073] [node 4] ICP22 created new session: [2:7571518619975583298:2073]
2025-11-11T17:19:13.074013Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518615680615990:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:13.087557Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:13.088343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518615680615980:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.088398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:13.088451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518615680615980:2076] [node 6] ICP22 created new session: [2:7571518619975583299:2076]
2025-11-11T17:19:13.088493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518619975583299:2076] [node 6] ICS09 handshake done sender: [2:7571518615680615990:2115] self: [2:7571518615680615989:0] peer: [6:7571518617503038190:0] socket: 251 qp: -1
2025-11-11T17:19:13.088525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518619975583299:2076] [node 6] ICS10 traffic start
2025-11-11T17:19:13.088567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:13.088915Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518615680615991:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:13.089111Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518611385648614:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518615680615991:2116] held: no
2025-11-11T17:19:13.089186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518611385648614:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.089230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:13.089267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518611385648614:2048] [node 1] ICP22 created new session: [2:7571518619975583301:2048]
2025-11-11T17:19:13.089305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518619975583301:2048] [node 1] ICS09 handshake done sender: [2:7571518615680615991:2116] self: [2:7571518615680615997:0] peer: [1:7571518616080293579:0] socket: 246 qp: -1
2025-11-11T17:19:13.089336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518619975583301:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:13.089359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:13.121771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518619975583298:2073] [node 4] ICS09 handshake done sender: [2:7571518615680615986:2114] self: [2:7571518615680615985:0] peer: [4:7571518612669868502:0] socket: 262 qp: -1
2025-11-11T17:19:13.136143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518619975583298:2073] [node 4] ICS10 traffic start
2025-11-11T17:19:13.136212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:13.144838Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518615680615996:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.144885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518615680615996:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.177972Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518615680615996:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:13.194562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518615680615981:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.210555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518619975583306:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62053
2025-11-11T17:19:13.210902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518615680615982:2073] [node 4] ICP35 event processing took too much time PendingConnection:278(168932012ns) {HandleHandshakeStatus:329(168930117ns) {DropIncomingHandshake:441(20860ns) DropOutgoingHandshake:456(348ns) SwitchToState:178(599ns) ProcessPendingSessionEvents:495(32922317ns) {ForwardSessionEventToSession:612(25138749ns) ForwardSessionEventToSession:612(7719688ns) ForwardSessionEventToSession:612(10660ns) ForwardSessionEventToSession:612(2294ns) ForwardSessionEventToSession:612(10946ns) ForwardSessionEventToSession:612(2645ns) ForwardSessionEventToSession:612(2765ns) ForwardSessionEventToSession:612(6578ns) ForwardSessionEventToSession:612(2681ns) ForwardSessionEventToSession:612(7744ns)}}}
2025-11-11T17:19:13.241650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:13.257710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518615680615981:2081] [node 5] ICP22 created new session: [2:7571518619975583307:2081]
2025-11-11T17:19:13.006850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:13.138662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518612669868415:2050] [node 3] ICP22 created new session: [4:7571518616964835809:2050]
2025-11-11T17:19:13.202506Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518617503038191:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.156175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518616564980900:2063] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:13.268436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518619975583303:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62053
2025-11-11T17:19:13.268741Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:13.269091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518615680615980:2076] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:13.269706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518619975583307:2081] [node 5] ICS09 handshake done sender: [2:7571518615680615996:2117] self: [2:7571518615680615995:0] peer: [5:7571518620859948270:0] socket: 289 qp: -1
2025-11-11T17:19:13.270129Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:13.278387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518619975583307:2081] [node 5] ICS10 traffic start
2025-11-11T17:19:13.278442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:13.288116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518620375260898:2082] [node 5] ICS09 handshake done sender: [1:7571518616080293588:2117] self: [1:7571518616080293583:0] peer: [5:7571518620859948275:0] socket: 240 qp: -1
2025-11-11T17:19:13.301666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518620375260898:2082] [node 5] ICS10 traffic start
2025-11-11T17:19:13.138704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518616964835809:2050] [node 3] ICS09 handshake done sender: [4:7571518616964835805:2121] self: [4:7571518616964835807:0] peer: [3:7571518614709856984:0] socket: 288 qp: -1
2025-11-11T17:19:13.018323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518614709856990:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17732
2025-11-11T17:19:13.173776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518620859948268:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.188528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518616564980883:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:13.188612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518620859948274:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:13.190072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518620859948273:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:13.199881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518620859948262:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:13.200065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518616564980916:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.200136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:13.200181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518616564980916:2069] [node 4] ICP22 created new session: [5:7571518620859948277:2069]
2025-11-11T17:19:13.200226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518620859948277:2069] [node 4] ICS09 handshake done sender: [5:7571518620859948262:2118] self: [5:7571518620859948269:0] peer: [4:7571518612669868494:0] socket: 247 qp: -1
2025-11-11T17:19:13.200252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518620859948277:2069] [node 4] ICS10 traffic start
2025-11-11T17:19:13.200287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:13.200611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518620859948276:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44340
2025-11-11T17:19:13.209829Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518620859948263:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:13.210200Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518620859948268:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:13.211068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518616564980885:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(88596335ns) {HandleHandshakeStatus:329(88593915ns) {DropIncomingHandshake:441(498ns) DropOutgoingHandshake:456(346ns) SwitchToState:178(394ns) ProcessPendingSessionEvents:495(1483ns)}}
2025-11-11T17:19:13.222081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518616564980884:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.222124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:13.250689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518616564980884:2049] [node 2] ICP22 created new session: [5:7571518620859948279:2049]
2025-11-11T17:19:13.250710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518616564980883:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.257826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:13.257877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518616564980883:2048] [node 1] ICP22 created new session: [5:7571518620859948280:2048]
2025-11-11T17:19:13.257917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518620859948280:2048] [node 1] ICS09 handshake done sender: [5:7571518620859948268:2120] self: [5:7571518620859948275:0] peer: [1:7571518616080293583:0] socket: 292 qp: -1
2025-11-11T17:19:13.257949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518620859948280:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:13.257985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:13.258370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518616564980883:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22342229ns) {HandleHandshakeStatus:329(22340675ns) {DropIncomingHandshake:441(1161ns) DropOutgoingHandshake:456(14834ns) SwitchToState:178(343ns) ProcessPendingSessionEvents:495(80724ns) {ForwardSessionEventToSession:612(41829ns) ForwardSessionEventToSession:612(1880ns) ForwardSessionEventToSession:612(5854ns) ForwardSessionEventToSession:612(2770ns) ForwardSessionEventToSession:612(2459ns) ForwardSessionEventToSession:612(4749ns) ForwardSessionEventToSession:612(5475ns) ForwardSessionEventToSession:612(4701ns) ForwardSessionEventToSession:612(2615ns) ForwardSessionEventToSession:612(2000ns)}}}
2025-11-11T17:19:13.268920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518620859948279:2049] [node 2] ICS09 handshake done sender: [5:7571518620859948263:2119] self: [5:7571518620859948270:0] peer: [2:7571518615680615995:0] socket: 236 qp: -1
2025-11-11T17:19:13.281800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518620859948279:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:13.281839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:13.138735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518616964835809:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:13.138789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:13.145065Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518612669868497:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.145098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518612669868497:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.165719Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518612669868497:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:13.205479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518612669868465:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.205944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518612669868431:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(187181159ns) {HandleHandshakeStatus:329(187178601ns) {DropIncomingHandshake:441(19384ns) DropOutgoingHandshake:456(551ns) SwitchToState:178(184ns) ProcessPendingSessionEvents:495(52011422ns) {ForwardSessionEventToSession:612(23947323ns) ForwardSessionEventToSession:612(6275ns) ForwardSessionEventToSession:612(7703985ns) ForwardSessionEventToSession:612(3883ns) ForwardSessionEventToSession:612(8243694ns) ForwardSessionEventToSession:612(5781ns) ForwardSessionEventToSession:612(1030015ns) ForwardSessionEventToSession:612(11039329ns) ForwardSessionEventToSession:612(3403ns) ForwardSessionEventToSession:612(3259ns)}}}
2025-11-11T17:19:13.261306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:13.268297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518612669868465:2066] [node 5] ICP22 created new session: [4:7571518616964835812:2066]
2025-11-11T17:19:13.281915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518616964835812:2066] [node 5] ICS09 handshake done sender: [4:7571518612669868497:2118] self: [4:7571518612669868494:0] peer: [5:7571518620859948269:0] socket: 248 qp: -1
2025-11-11T17:19:13.297953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518616964835812:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:13.202674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518613208070810:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:13.210861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518617503038197:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:13.216516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518613208070809:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.216581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:13.216620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518613208070809:2049] [node 2] ICP22 created new session: [6:7571518617503038200:2049]
2025-11-11T17:19:13.216658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518617503038200:2049] [node 2] ICS09 handshake done sender: [6:7571518613208070887:2116] self: [6:7571518617503038190:0] peer: [2:7571518615680615989:0] socket: 252 qp: -1
2025-11-11T17:19:13.216689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518617503038200:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:13.216720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:13.217110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518613208070843:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.217161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:13.217194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518613208070843:2068] [node 4] ICP22 created new session: [6:7571518617503038202:2068]
2025-11-11T17:19:13.217225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518617503038202:2068] [node 4] ICS09 handshake done sender: [6:7571518613208070879:2113] self: [6:7571518613208070886:0] peer: [4:7571518612669868486:0] socket: 267 qp: -1
2025-11-11T17:19:13.217253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518617503038202:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:13.217277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:13.217450Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518613208070891:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:13.233944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518613208070808:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.234009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:13.234054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518613208070808:2048] [node 1] ICP22 created new session: [6:7571518617503038205:2048]
2025-11-11T17:19:13.234100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518617503038205:2048] [node 1] ICS09 handshake done sender: [6:7571518613208070891:2118] self: [6:7571518617503038193:0] peer: [1:7571518616080293587:0] socket: 278 qp: -1
2025-11-11T17:19:13.234141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518617503038205:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:13.234171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:13.234899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518617503038204:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17422
2025-11-11T17:19:13.253934Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518617503038191:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:13.259147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518613208070810:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.259212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:13.259249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518613208070810:2050] [node 3] ICP22 created new session: [6:7571518617503038207:2050]
2025-11-11T17:19:13.259286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518617503038207:2050] [node 3] ICS09 handshake done sender: [6:7571518617503038191:2121] self: [6:7571518617503038199:0] peer: [3:7571518614709856986:0] socket: 237 qp: -1
2025-11-11T17:19:13.259315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518617503038207:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:13.259358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:13.313875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:13.314291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518616080293563:2082] [node 5] ICP35 event processing took too much time PendingConnection:278(102421537ns) {HandleHandshakeStatus:329(102418967ns) {DropIncomingHandshake:441(19030ns) DropOutgoingHandshake:456(299ns) SwitchToState:178(947ns) ProcessPendingSessionEvents:495(88229ns) {ForwardSessionEventToSession:612(44606ns) ForwardSessionEventToSession:612(5929ns) ForwardSessionEventToSession:612(3246ns) ForwardSessionEventToSession:612(2156ns) ForwardSessionEventToSession:612(4670ns) ForwardSessionEventToSession:612(2236ns) ForwardSessionEventToSession:612(6108ns) ForwardSessionEventToSession:612(5074ns) ForwardSessionEventToSession:612(1768ns) ForwardSessionEventToSession:612(4518ns)}}}
2025-11-11T17:19:13.027976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518614709856904:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(21086001ns) {HandleHandshakeStatus:329(21082034ns) {DropIncomingHandshake:441(227ns) DropOutgoingHandshake:456(3744ns) SwitchToState:178(110ns) ProcessPendingSessionEvents:495(13573772ns) {ForwardSessionEventToSession:612(13505426ns) ForwardSessionEventToSession:612(9801ns) ForwardSessionEventToSession:612(3765ns) ForwardSessionEventToSession:612(5753ns) ForwardSessionEventToSession:612(17754ns) ForwardSessionEventToSession:612(3926ns) ForwardSessionEventToSession:612(3533ns) ForwardSessionEventToSession:612(6256ns) ForwardSessionEventToSession:612(2956ns) ForwardSessionEventToSession:612(5282ns)}}}
2025-11-11T17:19:13.028073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518614709856991:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17732
2025-11-11T17:19:13.028256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518614709856923:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.028305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:13.028349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518614709856923:2064] [node 5] ICP22 created new session: [3:7571518619004824290:2064]
2025-11-11T17:19:13.028390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518619004824290:2064] [node 5] ICS09 handshake done sender: [3:7571518614709856978:2114] self: [3:7571518614709856977:0] peer: [5:7571518616564980962:0] socket: 258 qp: -1
2025-11-11T17:19:13.028418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518619004824290:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:13.028446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:13.028754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518614709856904:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:13.038270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518614709856980:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:13.075392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518614709856903:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:13.075428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:13.075463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518614709856903:2048] [node 1] ICP22 created new session: [3:7571518619004824294:2048]
2025-11-11T17:19:13.075505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518619004824294:2048] [node 1] ICS09 handshake done sender: [3:7571518614709856980:2115] self: [3:7571518614709856988:0] peer: [1:7571518616080293568:0] socket: 257 qp: -1
2025-11-11T17:19:13.075529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518619004824294:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:13.075559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:13.094935Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518614709856991:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.109359Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518614709856985:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.109380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518614709856985:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.130034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518614709856976:2063] [node 4] ICP35 event processing took too much time PendingConnection:278(20085579ns) {IncomingHandshake:261(20084141ns) {DropIncomingHandshake:441(293ns) PrepareNewSessionHandshake:476(1344ns) {SwitchToState:178(358ns)} GenerateSessionVirtualId:988(820ns) IssueIncomingHandshakeReply:193(6480ns)}}
2025-11-11T17:19:13.134330Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518614709856985:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:13.160964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518614709856976:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.161086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:13.161128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518614709856976:2063] [node 4] ICP22 created new session: [3:7571518619004824298:2063]
2025-11-11T17:19:13.161170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518619004824298:2063] [node 4] ICS09 handshake done sender: [3:7571518614709856985:2117] self: [3:7571518614709856984:0] peer: [4:7571518616964835807:0] socket: 271 qp: -1
2025-11-11T17:19:13.161202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518619004824298:2063] [node 4] ICS10 traffic start
2025-11-11T17:19:13.161248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:13.194347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518619004824300:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17732
2025-11-11T17:19:13.211412Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518614709856987:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.211438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518614709856987:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:13.218363Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518614709856987:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:13.218575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518614709856967:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:13.218626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:13.218660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518614709856967:2065] [node 6] ICP22 created new session: [3:7571518619004824302:2065]
2025-11-11T17:19:13.218699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518619004824302:2065] [node 6] ICS09 handshake done sender: [3:7571518614709856987:2118] self: [3:7571518614709856986:0] peer: [6:7571518617503038199:0] socket: 287 qp: -1
2025-11-11T17:19:13.218727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518619004824302:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:13.218770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:13.261231Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518619004824300:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:13.297903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518614709856967:2065] [node 6] ICP18 handshake (actor [3:7571518619004824300:2121]) is too old
2025-11-11T17:19:13.317727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [3:7571518614709856967:2065] [node 6] ICP35 event processing took too much time StateWork:293(22001172ns) {IncomingHandshake:261(6030625ns)}
2025-11-11T17:19:13.317988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518615680615981:2081] [node 5] ICP35 event processing took too much time PendingConnection:278(123207773ns) {HandleHandshakeStatus:329(123205363ns) {DropIncomingHandshake:441(654ns) DropOutgoingHandshake:456(607ns) SwitchToState:178(700ns) ProcessPendingSessionEvents:495(9231575ns) {ForwardSessionEventToSession:612(9180417ns) ForwardSessionEventToSession:612(5625ns) ForwardSessionEventToSession:612(2946ns) ForwardSessionEventToSession:612(11401ns) ForwardSessionEventToSession:612(2215ns) ForwardSessionEventToSession:612(4515ns) ForwardSessionEventToSession:612(1949ns) ForwardSessionEventToSession:612(2498ns) ForwardSessionEventToSession:612(5187ns) ForwardSessionEventToSession:612(1926ns)}}}
2025-11-11T17:19:13.309693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:13.323365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518612669868465:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(125404845ns) {HandleHandshakeStatus:329(125402741ns) {DropIncomingHandshake:441(14477ns) DropOutgoingHandshake:456(2763ns) SwitchToState:178(409ns) ProcessPendingSessionEvents:495(5104746ns) {ForwardSessionEventToSession:612(5054242ns) ForwardSessionEventToSession:612(5078ns) ForwardSessionEventToSession:612(9419ns) ForwardSessionEventToSession:612(5420ns) ForwardSessionEventToSession:612(2041ns) ForwardSessionEventToSession:612(5265ns) ForwardSessionEventToSession:612(2541ns) ForwardSessionEventToSession:612(5345ns) ForwardSessionEventToSession:612(2103ns) ForwardSessionEventToSession:612(1860ns)}}}
2025-11-11T17:19:13.307073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518616564980884:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(94963448ns) {HandleHandshakeStatus:329(94961202ns) {DropIncomingHandshake:441(952ns) DropOutgoingHandshake:456(6283ns) SwitchToState:178(314ns) ProcessPendingSessionEvents:495(3019222ns) {ForwardSessionEventToSession:612(2973750ns) ForwardSessionEventToSession:612(6807ns) ForwardSessionEventToSession:612(2448ns) ForwardSessionEventToSession:612(4160ns) ForwardSessionEventToSession:612(3077ns) ForwardSessionEventToSession:612(3177ns) ForwardSessionEventToSession:612(4929ns) ForwardSessionEventToSession:612(2467ns) ForwardSessionEventToSession:612(4210ns) ForwardSessionEventToSession:612(4133ns)}}}
2025-11-11T17:19:13.329982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518615680615981:2081] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:13.415812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:13.416039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518619975583301:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-11-11T17:19:13.416078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518611385648614:2048] [node 1] ICP30 unregister session Session# [2:7571518619975583301:2048] VirtualId# [2:7571518615680615997:0]
2025-11-11T17:19:13.416112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518619975583301:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.416193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:13.416228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:13.417713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:13.417906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518619004824294:2048] [node 1] ICS01 socket: 257 reason# EndOfStream
2025-11-11T17:19:13.417937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518614709856903:2048] [node 1] ICP30 unregister session Session# [3:7571518619004824294:2048] VirtualId# [3:7571518614709856988:0]
2025-11-11T17:19:13.417963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518619004824294:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.418016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:13.418050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:13.429233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:13.429449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518617503038205:2048] [node 1] ICS01 socket: 278 reason# EndOfStream
2025-11-11T17:19:13.429484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518613208070808:2048] [node 1] ICP30 unregister session Session# [6:7571518617503038205:2048] VirtualId# [6:7571518617503038193:0]
2025-11-11T17:19:13.429516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518617503038205:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.429580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:13.429824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:13.429703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:13.429927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518612669868492:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-11-11T17:19:13.429973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518612669868413:2048] [node 1] ICP30 unregister session Session# [4:7571518612669868492:2048] VirtualId# [4:7571518612669868487:0]
2025-11-11T17:19:13.429997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518612669868492:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.430068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:13.430103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:13.430175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:13.430356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518620859948280:2048] [node 1] ICS07 socket disconnect 292 reason# EndOfStream
2025-11-11T17:19:13.430380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518620859948280:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.430420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:13.430455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518620859948280:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:13.431304Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Couldn't connect to any resolved address
2025-11-11T17:19:13.431537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518616564980883:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:13.431574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518620859948280:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:13.431600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518616564980883:2048] [node 1] ICP30 unregister session Session# [5:7571518620859948280:2048] VirtualId# [5:7571518620859948275:0]
2025-11-11T17:19:13.431635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:13.431659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518616564980883:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:13.431672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3973) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:13.445668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518616564980883:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:13.503298Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:13.503498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518619004824288:2049] [node 2] ICS01 socket: 245 reason# EndOfStream
2025-11-11T17:19:13.503536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518614709856904:2049] [node 2] ICP30 unregister session Session# [3:7571518619004824288:2049] VirtualId# [3:7571518614709856982:0]
2025-11-11T17:19:13.503569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518619004824288:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.503642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:13.503687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:13.513858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:13.514069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518616964835803:2049] [node 2] ICS01 socket: 263 reason# EndOfStream
2025-11-11T17:19:13.514102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518612669868414:2049] [node 2] ICP30 unregister session Session# [4:7571518616964835803:2049] VirtualId# [4:7571518612669868502:0]
2025-11-11T17:19:13.514130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518616964835803:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.513814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:13.514007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518617503038200:2049] [node 2] ICS01 socket: 252 reason# EndOfStream
2025-11-11T17:19:13.514046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518613208070809:2049] [node 2] ICP30 unregister session Session# [6:7571518617503038200:2049] VirtualId# [6:7571518617503038190:0]
2025-11-11T17:19:13.514074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518617503038200:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.514145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:13.514184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:13.514253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:13.514441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518620859948279:2049] [node 2] ICS01 socket: 236 reason# EndOfStream
2025-11-11T17:19:13.514470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518616564980884:2049] [node 2] ICP30 unregister session Session# [5:7571518620859948279:2049] VirtualId# [5:7571518620859948270:0]
2025-11-11T17:19:13.514496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518620859948279:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.514537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:13.514569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:13.514207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:13.514251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:13.658664Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:13.658802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518617503038207:2050] [node 3] ICS01 socket: 237 reason# EndOfStream
2025-11-11T17:19:13.658832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518613208070810:2050] [node 3] ICP30 unregister session Session# [6:7571518617503038207:2050] VirtualId# [6:7571518617503038199:0]
2025-11-11T17:19:13.658851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518617503038207:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.658890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:13.658920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:13.657795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:13.658020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518616964835809:2050] [node 3] ICS01 socket: 288 reason# EndOfStream
2025-11-11T17:19:13.658068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518612669868415:2050] [node 3] ICP30 unregister session Session# [4:7571518616964835809:2050] VirtualId# [4:7571518616964835807:0]
2025-11-11T17:19:13.658100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518616964835809:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.658187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:13.658229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:13.658319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:13.658471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518620859948267:2050] [node 3] ICS01 socket: 261 reason# EndOfStream
2025-11-11T17:19:13.658507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518616564980885:2050] [node 3] ICP30 unregister session Session# [5:7571518620859948267:2050] VirtualId# [5:7571518616564980962:0]
2025-11-11T17:19:13.658528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518620859948267:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.658568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:13.658597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:13.754302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:13.754485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518620859948277:2069] [node 4] ICS01 socket: 247 reason# EndOfStream
2025-11-11T17:19:13.754512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518616564980916:2069] [node 4] ICP30 unregister session Session# [5:7571518620859948277:2069] VirtualId# [5:7571518620859948269:0]
2025-11-11T17:19:13.754535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518620859948277:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.754595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:13.754628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:13.753781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:13.754009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518617503038202:2068] [node 4] ICS01 socket: 267 reason# EndOfStream
2025-11-11T17:19:13.754047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518613208070843:2068] [node 4] ICP30 unregister session Session# [6:7571518617503038202:2068] VirtualId# [6:7571518613208070886:0]
2025-11-11T17:19:13.754082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518617503038202:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.754167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:13.754208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:13.829761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:13.829968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518617503038194:2061] [node 5] ICS07 socket disconnect 253 reason# EndOfStream
2025-11-11T17:19:13.830009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518617503038194:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:13.830113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:13.830156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518617503038194:2061] [node 5] ICS15 start handshake
2025-11-11T17:19:13.830802Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13485) Couldn't connect to any resolved address
2025-11-11T17:19:13.831039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518613208070851:2061] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13485) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:13.831089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518617503038194:2061] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:13.831123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518613208070851:2061] [node 5] ICP30 unregister session Session# [6:7571518617503038194:2061] VirtualId# [6:7571518613208070888:0]
2025-11-11T17:19:13.831169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:19:13.831199Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518613208070851:2061] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:13.831211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:13.845778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518613208070851:2061] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:14.061893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518621935013134:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:14.061940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518621935013135:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:14.189889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518625369164061:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:14.189937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518625369164062:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:14.241917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518621994031204:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:14.241952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518621994031205:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:14.298053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518621704832277:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:14.298086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518621704832278:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:14.298108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518621704832279:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:14.350099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518624405287156:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:14.350134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518624405287157:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:14.350154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518624405287158:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:14.626127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518625369164082:2066] [node 6] ICP01 ready to work
2025-11-11T17:19:14.375272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518623939814588:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:14.375317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518623939814589:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:14.375344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518623939814590:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:14.670755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518623939814601:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.722215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518621704832301:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.702538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518623939814606:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:14.804668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518623939814610:2065] [node 5] ICP01 ready to work
2025-11-11T17:19:14.765739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518621935013197:2079] [node 5] ICP01 ready to work
2025-11-11T17:19:14.790784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518625369164131:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.764207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518621994031252:2062] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.900171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518623939814626:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:14.802266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518621935013198:2077] [node 6] ICP01 ready to work
2025-11-11T17:19:14.808871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518621994031265:2065] [node 4] ICP01 ready to work
2025-11-11T17:19:14.874554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518624405287189:2075] [node 6] ICP01 ready to work
2025-11-11T17:19:14.839014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518621704832301:2071] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.947243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518623939814610:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(142304440ns) {Configure:104(142300774ns) {ProcessConfigured:135(21261466ns) {StartInitialHandshake:157(21258786ns) {DropHandshakes:469(2178ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(377ns)} GenerateSessionVirtualId:988(1100ns) PrepareNewSessionHandshake:476(2943ns) {SwitchToState:178(1351ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.828885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518625369164125:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2087
2025-11-11T17:19:14.849931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518621994031268:2061] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.846019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518621704832335:2069] [node 6] ICP01 ready to work
2025-11-11T17:19:14.845863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518625369164134:2065] [node 4] ICP01 ready to work
2025-11-11T17:19:14.805915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518621935013202:2080] [node 4] ICP01 ready to work
2025-11-11T17:19:14.961667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518623939814606:2062] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:14.972350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518623939814589:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:14.981270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518623939814611:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:14.939477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518624405287207:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:14.856133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518621935013206:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58944
2025-11-11T17:19:14.860755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518621935013206:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:14.913712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518621935013208:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58944
2025-11-11T17:19:14.942229Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518621935013208:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:14.993212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518621994031265:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(153674821ns) {Configure:104(153669932ns) {ProcessConfigured:135(79475ns) {StartInitialHandshake:157(76943ns) {DropHandshakes:469(2699ns) {DropIncomingHandshake:441(612ns) DropOutgoingHandshake:456(474ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:476(2183ns) {SwitchToState:178(878ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.992437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518623939814626:2079] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:14.948016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518625369164137:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2087
2025-11-11T17:19:14.849808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518621704832339:2068] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.946622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518621704832342:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
2025-11-11T17:19:14.961615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518625369164062:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(199852098ns) {Configure:104(199848344ns) {ProcessConfigured:135(50528906ns) {StartInitialHandshake:157(50524797ns) {DropHandshakes:469(21504626ns) {DropIncomingHandshake:441(607ns) DropOutgoingHandshake:456(1155ns)} GenerateSessionVirtualId:988(1658ns) PrepareNewSessionHandshake:476(2845ns) {SwitchToState:178(1204ns)}}}}}
2025-11-11T17:19:14.987121Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518625369164137:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:14.989652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518625369164125:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.001841Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518625369164084:2069] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.001876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518625369164084:2069] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.021756Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518625369164084:2069] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:15.022139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518625369164082:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.022201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:15.022255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518625369164082:2066] [node 6] ICP22 created new session: [2:7571518629664131436:2066]
2025-11-11T17:19:15.022341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518629664131436:2066] [node 6] ICS09 handshake done sender: [2:7571518625369164084:2069] self: [2:7571518625369164083:0] peer: [6:7571518623939814667:0] socket: 240 qp: -1
2025-11-11T17:19:15.022386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518629664131436:2066] [node 6] ICS10 traffic start
2025-11-11T17:19:15.022426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:14.993469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518623939814666:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:15.012447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518626288998568:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-11-11T17:19:15.025041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518624405287209:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:14.989141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518621704832342:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.018009Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518621935013200:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.001307Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518623939814611:2066] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.001434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518623939814588:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.007942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518623939814610:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.010755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518623939814590:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(277828485ns) {Configure:104(277824109ns) {ProcessConfigured:135(65531ns) {StartInitialHandshake:157(62009ns) {DropHandshakes:469(2869ns) {DropIncomingHandshake:441(625ns) DropOutgoingHandshake:456(436ns)} GenerateSessionVirtualId:988(1016ns) PrepareNewSessionHandshake:476(1611ns) {SwitchToState:178(604ns)}}}}}
2025-11-11T17:19:15.033923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518628234781970:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:15.034232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518628234781969:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:15.049706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518623939814666:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.063474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518623939814601:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.012627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518626288998569:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.025089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518624405287209:2093] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.018048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518621935013200:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.013364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518625999799645:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
2025-11-11T17:19:15.078844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518625369164131:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(213181184ns) {Configure:104(213176407ns) {ProcessConfigured:135(81381ns) {StartInitialHandshake:157(78796ns) {DropHandshakes:469(2391ns) {DropIncomingHandshake:441(853ns) DropOutgoingHandshake:456(325ns)} GenerateSessionVirtualId:988(497ns) PrepareNewSessionHandshake:476(1710ns) {SwitchToState:178(873ns)}}}}}
2025-11-11T17:19:15.012931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518626288998568:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.106181Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518623939814606:2062] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:15.106634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518623939814589:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.106695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:15.106738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518623939814589:2049] [node 2] ICP22 created new session: [6:7571518628234781972:2049]
2025-11-11T17:19:15.106794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518628234781972:2049] [node 2] ICS09 handshake done sender: [6:7571518623939814606:2062] self: [6:7571518623939814667:0] peer: [2:7571518625369164083:0] socket: 245 qp: -1
2025-11-11T17:19:15.106828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518628234781972:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:15.106864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:15.025965Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518621935013200:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.058331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254527:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.114310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518629664131442:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2087
2025-11-11T17:19:15.033368Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518626288998569:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.149035Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518625369164135:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.149073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518625369164135:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.150321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518625369164135:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.151251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518629664131444:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2087
2025-11-11T17:19:15.151523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518625369164062:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.151588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:15.151637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518625369164062:2049] [node 3] ICP22 created new session: [2:7571518629664131445:2049]
2025-11-11T17:19:15.151686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518629664131445:2049] [node 3] ICS09 handshake done sender: [2:7571518625369164135:2113] self: [2:7571518625369164132:0] peer: [3:7571518626288998585:0] socket: 257 qp: -1
2025-11-11T17:19:15.151733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518629664131445:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:15.151764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:15.046798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518621935013197:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(208954598ns) {Configure:104(208941579ns) {ProcessConfigured:135(71579ns) {StartInitialHandshake:157(68284ns) {DropHandshakes:469(2904ns) {DropIncomingHandshake:441(635ns) DropOutgoingHandshake:456(386ns)} GenerateSessionVirtualId:988(1146ns) PrepareNewSessionHandshake:476(1248ns) {SwitchToState:178(460ns)}}}}}
2025-11-11T17:19:15.062668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518621704832277:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(350006656ns) {Configure:104(350001600ns) {ProcessConfigured:135(7491831ns) {StartInitialHandshake:157(7488708ns) {DropHandshakes:469(2743ns) {DropIncomingHandshake:441(710ns) DropOutgoingHandshake:456(414ns)} GenerateSessionVirtualId:988(931ns) PrepareNewSessionHandshake:476(3092ns) {SwitchToState:178(1272ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.058890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518624405287209:2093] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:15.130080Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518623939814611:2066] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:15.141766Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:15.146847Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:15.159034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518623939814589:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:15.159131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518623939814588:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518623939814611:2066] held: no
2025-11-11T17:19:15.159274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518628234781974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:15.083380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518621935013135:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(453753011ns) {Configure:104(453746335ns) {ProcessConfigured:135(10026757ns) {StartInitialHandshake:157(10024091ns) {DropHandshakes:469(2658ns) {DropIncomingHandshake:441(511ns) DropOutgoingHandshake:456(519ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:476(3406ns) {SwitchToState:178(1838ns)}}}}}
2025-11-11T17:19:15.104032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518624405287156:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(457920923ns) {Configure:104(437829806ns) {ProcessConfigured:135(20106540ns) {StartInitialHandshake:157(20103800ns) {DropHandshakes:469(1041ns) {DropIncomingHandshake:441(376ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:476(2079ns) {SwitchToState:178(121ns)}}}}}
2025-11-11T17:19:15.122695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518624405287207:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(143087523ns) {Configure:104(143079275ns) {ProcessConfigured:135(60468ns) {StartInitialHandshake:157(58282ns) {DropHandshakes:469(2076ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:476(1754ns) {SwitchToState:178(514ns)}}}}}
2025-11-11T17:19:15.157904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518624405287189:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.175055Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518628700254527:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.175215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518624405287156:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.158128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518625369164134:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(292248544ns) {Configure:104(292242251ns) {ProcessConfigured:135(57305ns) {StartInitialHandshake:157(54985ns) {DropHandshakes:469(822ns) {DropIncomingHandshake:441(231ns) DropOutgoingHandshake:456(222ns)} GenerateSessionVirtualId:988(881ns) PrepareNewSessionHandshake:476(655ns) {SwitchToState:178(89ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.111176Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518621935013204:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.062854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518621704832277:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.082191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518621994031204:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(352173092ns) {Configure:104(352167919ns) {ProcessConfigured:135(64055ns) {StartInitialHandshake:157(61316ns) {DropHandshakes:469(1195ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(271ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:476(1401ns) {SwitchToState:178(487ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.161976Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518629664131444:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.083003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518626288998575:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-11-11T17:19:15.182813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:15.111212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518621935013204:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.083968Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518625999799645:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.182542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518628234781975:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.191842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254535:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.185703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518625369164061:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.125683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518621935013198:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.101887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518621704832344:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.103474Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518626288998575:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.189722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518623939814588:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.125763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:15.221741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518624405287189:2075] [node 6] ICP22 created new session: [5:7571518628700254534:2075]
2025-11-11T17:19:15.123119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518621994031252:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(309175013ns) {Configure:104(309169343ns) {ProcessConfigured:135(18558821ns) {StartInitialHandshake:157(18557248ns) {DropHandshakes:469(1654ns) {DropIncomingHandshake:441(377ns) DropOutgoingHandshake:456(437ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:476(2236ns) {SwitchToState:178(896ns)}}}}}
2025-11-11T17:19:15.101931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518621704832344:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.189895Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518629664131442:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.202788Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518623939814626:2079] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:15.125818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518621935013198:2077] [node 6] ICP22 created new session: [1:7571518626229980513:2077]
2025-11-11T17:19:15.190041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518625369164062:2049] [node 3] ICP18 handshake (actor [2:7571518629664131442:2116]) is too old
2025-11-11T17:19:15.234139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254532:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.105704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518621704832279:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(155894725ns) {Configure:104(155880612ns) {ProcessConfigured:135(40970263ns) {StartInitialHandshake:157(40966834ns) {DropHandshakes:469(9150571ns) {DropIncomingHandshake:441(536ns) DropOutgoingHandshake:456(1613ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:476(4131ns) {SwitchToState:178(2286ns)}}}}}
2025-11-11T17:19:15.135662Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518621994031271:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.205310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.125871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518626229980513:2077] [node 6] ICS09 handshake done sender: [1:7571518621935013200:2110] self: [1:7571518621935013199:0] peer: [6:7571518628234781965:0] socket: 246 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.234269Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518628700254527:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:15.135721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518621994031271:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.259968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518629664131452:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2087
2025-11-11T17:19:15.222295Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518623939814666:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:15.110242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518621704832279:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.125223Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518621704832344:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:15.142150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518621704832278:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(379915674ns) {Configure:104(379904434ns) {ProcessConfigured:135(34556881ns) {StartInitialHandshake:157(34554066ns) {DropHandshakes:469(987ns) {DropIncomingHandshake:441(175ns) DropOutgoingHandshake:456(212ns)} GenerateSessionVirtualId:988(346ns) PrepareNewSessionHandshake:476(2219ns) {SwitchToState:178(149ns)}}}}}
2025-11-11T17:19:15.214200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518625999799656:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
2025-11-11T17:19:15.242170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518621704832339:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(252716326ns) {Configure:104(240906949ns) {ProcessConfigured:135(51842ns) {StartInitialHandshake:157(49451ns) {DropHandshakes:469(2053ns) {DropIncomingHandshake:441(455ns) DropOutgoingHandshake:456(410ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:476(4051ns) {SwitchToState:178(587ns)}}}}}
2025-11-11T17:19:15.242212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518621704832335:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.242262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:15.242307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518621704832335:2069] [node 6] ICP22 created new session: [4:7571518625999799661:2069]
2025-11-11T17:19:15.242348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518625999799661:2069] [node 6] ICS09 handshake done sender: [4:7571518621704832344:2112] self: [4:7571518621704832343:0] peer: [6:7571518628234781971:0] socket: 258 qp: -1
2025-11-11T17:19:15.242381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518625999799661:2069] [node 6] ICS10 traffic start
2025-11-11T17:19:15.242413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:15.245817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518625999799657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
2025-11-11T17:19:15.254813Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518621704832342:2110] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:15.261169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518621704832277:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.273832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518625999799658:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
2025-11-11T17:19:15.274725Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:15.288133Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518625999799645:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:15.288396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518621704832279:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518625999799645:2113] held: no
2025-11-11T17:19:15.289947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518625999799664:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
2025-11-11T17:19:15.297379Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518625999799658:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.297427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518621704832279:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.300652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:15.300759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:15.300803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518621704832279:2050] [node 3] ICP22 created new session: [4:7571518625999799665:2050]
2025-11-11T17:19:15.300841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518625999799665:2050] [node 3] ICS09 handshake done sender: [4:7571518625999799645:2113] self: [4:7571518625999799654:0] peer: [3:7571518621994031270:0] socket: 266 qp: -1
2025-11-11T17:19:15.300875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518625999799665:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:15.300904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:15.301300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518625999799664:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.306433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518621704832278:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.316838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518621704832277:2048] [node 1] ICP22 created new session: [4:7571518625999799666:2048]
2025-11-11T17:19:15.334361Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518625999799660:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.334385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518625999799660:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.335292Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518625999799660:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:15.335545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518621704832339:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.335617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:15.335666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518621704832339:2068] [node 5] ICP22 created new session: [4:7571518625999799672:2068]
2025-11-11T17:19:15.335723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518625999799672:2068] [node 5] ICS09 handshake done sender: [4:7571518625999799660:2120] self: [4:7571518625999799659:0] peer: [5:7571518628700254546:0] socket: 254 qp: -1
2025-11-11T17:19:15.335751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518625999799672:2068] [node 5] ICS10 traffic start
2025-11-11T17:19:15.335807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:15.337796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518625999799666:2048] [node 1] ICS09 handshake done sender: [4:7571518621704832342:2110] self: [4:7571518625999799650:0] peer: [1:7571518621935013203:0] socket: 250 qp: -1
2025-11-11T17:19:15.357690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518625999799666:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:15.357755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:15.358250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518621704832277:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(96838497ns) {HandleHandshakeStatus:329(96836692ns) {DropIncomingHandshake:441(524ns) DropOutgoingHandshake:456(3803ns) SwitchToState:178(389ns) ProcessPendingSessionEvents:495(114283ns) {ForwardSessionEventToSession:612(59064ns) ForwardSessionEventToSession:612(11047ns) ForwardSessionEventToSession:612(6322ns) ForwardSessionEventToSession:612(6291ns) ForwardSessionEventToSession:612(3732ns) ForwardSessionEventToSession:612(2193ns) ForwardSessionEventToSession:612(6441ns) ForwardSessionEventToSession:612(3088ns) ForwardSessionEventToSession:612(2310ns) ForwardSessionEventToSession:612(5107ns)}}}
2025-11-11T17:19:15.370974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518625999799675:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7696
2025-11-11T17:19:15.371831Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518625999799658:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:15.372072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518621704832278:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.372146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:15.372190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518621704832278:2049] [node 2] ICP22 created new session: [4:7571518625999799676:2049]
2025-11-11T17:19:15.372231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518625999799676:2049] [node 2] ICS09 handshake done sender: [4:7571518625999799658:2119] self: [4:7571518625999799669:0] peer: [2:7571518629664131447:0] socket: 299 qp: -1
2025-11-11T17:19:15.372262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518625999799676:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:15.372288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:15.145213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518621994031205:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(380648125ns) {Configure:104(380641789ns) {ProcessConfigured:135(27967348ns) {StartInitialHandshake:157(27964717ns) {DropHandshakes:469(1104ns) {DropIncomingHandshake:441(270ns) DropOutgoingHandshake:456(210ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:476(3309ns) {SwitchToState:178(1499ns)}}}}}
2025-11-11T17:19:15.236064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518628700254534:2075] [node 6] ICS09 handshake done sender: [5:7571518624405287209:2093] self: [5:7571518624405287208:0] peer: [6:7571518628234781966:0] socket: 255 qp: -1
2025-11-11T17:19:15.125902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518626229980513:2077] [node 6] ICS10 traffic start
2025-11-11T17:19:15.222480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518623939814610:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.297299Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518629664131439:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.125957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:15.146257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518621994031268:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(206620362ns) {Configure:104(206615031ns) {ProcessConfigured:135(57572ns) {StartInitialHandshake:157(55210ns) {DropHandshakes:469(2744ns) {DropIncomingHandshake:441(611ns) DropOutgoingHandshake:456(439ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:476(2338ns) {SwitchToState:178(1061ns)}}}}}
2025-11-11T17:19:15.297330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518629664131439:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.222525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:15.243710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518628700254534:2075] [node 6] ICS10 traffic start
2025-11-11T17:19:15.222576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518623939814610:2065] [node 5] ICP22 created new session: [6:7571518628234781978:2065]
2025-11-11T17:19:15.222620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518628234781978:2065] [node 5] ICS09 handshake done sender: [6:7571518623939814626:2079] self: [6:7571518628234781966:0] peer: [5:7571518624405287208:0] socket: 256 qp: -1
2025-11-11T17:19:15.222659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518628234781978:2065] [node 5] ICS10 traffic start
2025-11-11T17:19:15.222696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:15.223523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518623939814601:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.223560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:15.223599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518623939814601:2059] [node 4] ICP22 created new session: [6:7571518628234781981:2059]
2025-11-11T17:19:15.223648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518628234781981:2059] [node 4] ICS09 handshake done sender: [6:7571518623939814666:2117] self: [6:7571518628234781971:0] peer: [4:7571518621704832343:0] socket: 261 qp: -1
2025-11-11T17:19:15.223671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518628234781981:2059] [node 4] ICS10 traffic start
2025-11-11T17:19:15.223709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:15.233927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518623939814588:2048] [node 1] ICP22 created new session: [6:7571518628234781977:2048]
2025-11-11T17:19:15.273969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518628234781977:2048] [node 1] ICS09 handshake done sender: [6:7571518623939814611:2066] self: [6:7571518628234781965:0] peer: [1:7571518621935013199:0] socket: 247 qp: -1
2025-11-11T17:19:15.282090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518628234781983:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:15.283506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518628234781977:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:15.297898Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518628234781983:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.298013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:15.317016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518623939814590:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.393669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518623939814588:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(203401636ns) {HandleHandshakeStatus:329(203399394ns) {DropIncomingHandshake:441(1603ns) DropOutgoingHandshake:456(399ns) SwitchToState:178(340ns) ProcessPendingSessionEvents:495(695ns)}}
2025-11-11T17:19:15.413916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518628234781986:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51255
2025-11-11T17:19:15.129879Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518621935013204:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:15.302664Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:15.243748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:15.254416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518624405287156:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.254481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:15.254529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518624405287156:2048] [node 1] ICP22 created new session: [5:7571518628700254539:2048]
2025-11-11T17:19:15.254571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518628700254539:2048] [node 1] ICS09 handshake done sender: [5:7571518628700254527:2113] self: [5:7571518628700254533:0] peer: [1:7571518626229980506:0] socket: 260 qp: -1
2025-11-11T17:19:15.254599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518628700254539:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:15.254629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:15.274032Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518628700254532:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.274185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518624405287157:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.287076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254543:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.287424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518624405287156:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32828871ns) {HandleHandshakeStatus:329(32826846ns) {DropIncomingHandshake:441(806ns) DropOutgoingHandshake:456(4400ns) SwitchToState:178(361ns) ProcessPendingSessionEvents:495(26186788ns) {ForwardSessionEventToSession:612(26144779ns) ForwardSessionEventToSession:612(4686ns) ForwardSessionEventToSession:612(5293ns) ForwardSessionEventToSession:612(3155ns) ForwardSessionEventToSession:612(2794ns) ForwardSessionEventToSession:612(4165ns) ForwardSessionEventToSession:612(2656ns) ForwardSessionEventToSession:612(6584ns) ForwardSessionEventToSession:612(2213ns) ForwardSessionEventToSession:612(2494ns)}}}
2025-11-11T17:19:15.307237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254544:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.307297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254545:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.315363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518624405287189:2075] [node 6] ICP35 event processing took too much time PendingConnection:278(157220949ns) {HandleHandshakeStatus:329(157218380ns) {DropIncomingHandshake:441(306ns) DropOutgoingHandshake:456(543ns) SwitchToState:178(529ns) ProcessPendingSessionEvents:495(14501246ns) {ForwardSessionEventToSession:612(14450997ns) ForwardSessionEventToSession:612(4978ns) ForwardSessionEventToSession:612(4536ns) ForwardSessionEventToSession:612(2396ns) ForwardSessionEventToSession:612(3119ns) ForwardSessionEventToSession:612(4466ns) ForwardSessionEventToSession:612(2715ns) ForwardSessionEventToSession:612(2459ns) ForwardSessionEventToSession:612(10371ns) ForwardSessionEventToSession:612(2447ns)}}}
2025-11-11T17:19:15.333815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518628700254544:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.333977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518624405287207:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.382104Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518628700254543:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.382177Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518628700254532:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:15.382732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518624405287158:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.398511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254538:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.398563Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:15.398954Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4126) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:15.401739Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:15.406239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518624405287207:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4126) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518628700254544:2120] held: no
2025-11-11T17:19:15.406327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518624405287157:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.406364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:15.406401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518624405287157:2049] [node 2] ICP22 created new session: [5:7571518628700254549:2049]
2025-11-11T17:19:15.406446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518628700254549:2049] [node 2] ICS09 handshake done sender: [5:7571518628700254532:2116] self: [5:7571518628700254542:0] peer: [2:7571518629664131438:0] socket: 252 qp: -1
2025-11-11T17:19:15.406473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518628700254549:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:15.406502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:15.406686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518624405287157:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:15.410038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254548:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.441714Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518628700254544:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:15.146334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518621994031205:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.148885Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518621994031271:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:15.191519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518626288998587:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-11-11T17:19:15.234681Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:15.243329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518621994031205:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518626288998569:2112] held: no
2025-11-11T17:19:15.243524Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518626288998569:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:15.243864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518626288998582:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-11-11T17:19:15.258278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518621994031265:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.258330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:15.258380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518621994031265:2065] [node 4] ICP22 created new session: [3:7571518626288998589:2065]
2025-11-11T17:19:15.258417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518626288998589:2065] [node 4] ICS09 handshake done sender: [3:7571518621994031271:2110] self: [3:7571518621994031270:0] peer: [4:7571518625999799654:0] socket: 265 qp: -1
2025-11-11T17:19:15.258449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518626288998589:2065] [node 4] ICS10 traffic start
2025-11-11T17:19:15.258478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:15.258824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518621994031205:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.258852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:15.258881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518621994031205:2049] [node 2] ICP22 created new session: [3:7571518626288998591:2049]
2025-11-11T17:19:15.258924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518626288998591:2049] [node 2] ICS09 handshake done sender: [3:7571518626288998569:2112] self: [3:7571518626288998585:0] peer: [2:7571518625369164132:0] socket: 264 qp: -1
2025-11-11T17:19:15.258950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518626288998591:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:15.142569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518626229980517:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58944
2025-11-11T17:19:15.161943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518621935013202:2080] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.161994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:15.162037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518621935013202:2080] [node 4] ICP22 created new session: [1:7571518626229980519:2080]
2025-11-11T17:19:15.175481Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518626229980507:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.175501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518626229980507:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.182969Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518626229980507:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:15.185624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518621935013134:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(484639250ns) {Configure:104(484635852ns) {ProcessConfigured:135(52668705ns) {StartInitialHandshake:157(52666322ns) {DropHandshakes:469(2483ns) {DropIncomingHandshake:441(609ns) DropOutgoingHandshake:456(387ns)} GenerateSessionVirtualId:988(878ns) PrepareNewSessionHandshake:476(1885ns) {SwitchToState:178(174ns)}}}}}
2025-11-11T17:19:15.189834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518626229980519:2080] [node 4] ICS09 handshake done sender: [1:7571518621935013204:2111] self: [1:7571518621935013203:0] peer: [4:7571518625999799650:0] socket: 249 qp: -1
2025-11-11T17:19:15.190982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518626229980521:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58944
2025-11-11T17:19:15.203018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518626229980512:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.203047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518626229980512:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.204984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518626229980522:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58944
2025-11-11T17:19:15.214638Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518626229980512:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:15.217680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518626229980519:2080] [node 4] ICS10 traffic start
2025-11-11T17:19:15.226184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:15.235980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518621935013197:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.236028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:15.236073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518621935013197:2079] [node 5] ICP22 created new session: [1:7571518626229980526:2079]
2025-11-11T17:19:15.245701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518626229980517:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.245771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518621935013134:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.247428Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518626229980521:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.267796Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:15.285631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518626229980526:2079] [node 5] ICS09 handshake done sender: [1:7571518626229980507:2114] self: [1:7571518626229980506:0] peer: [5:7571518628700254533:0] socket: 270 qp: -1
2025-11-11T17:19:15.286081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518621935013202:2080] [node 4] ICP35 event processing took too much time PendingConnection:278(136121899ns) {HandleHandshakeStatus:329(131768067ns) {DropIncomingHandshake:441(954ns) DropOutgoingHandshake:456(899ns) SwitchToState:178(499ns) ProcessPendingSessionEvents:495(108394ns) {ForwardSessionEventToSession:612(59450ns) ForwardSessionEventToSession:612(7538ns) ForwardSessionEventToSession:612(2190ns) ForwardSessionEventToSession:612(3247ns) ForwardSessionEventToSession:612(4944ns) ForwardSessionEventToSession:612(2711ns) ForwardSessionEventToSession:612(4990ns) ForwardSessionEventToSession:612(3093ns) ForwardSessionEventToSession:612(2508ns) ForwardSessionEventToSession:612(7485ns)}}}
2025-11-11T17:19:15.286540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518621935013202:2080] [node 4] ICP18 handshake (actor [1:7571518626229980517:2117]) is too old
2025-11-11T17:19:15.290392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518626229980526:2079] [node 5] ICS10 traffic start
2025-11-11T17:19:15.293795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:15.296123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518626229980511:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.296147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518626229980511:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.297319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:15.297768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518626229980511:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:15.298389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518621935013134:2048] [node 2] ICP22 created new session: [1:7571518626229980527:2048]
2025-11-11T17:19:15.258994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:15.438584Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518628234781983:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:15.329993Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518629664131439:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:15.457792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518624405287207:2064] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:15.260310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518626288998588:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-11-11T17:19:15.275555Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518626288998588:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.287747Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518626288998582:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.287869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518621994031204:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:15.306149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518626288998597:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-11-11T17:19:15.326193Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518626288998582:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:15.365863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518621994031204:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.365930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:15.365976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518621994031204:2048] [node 1] ICP22 created new session: [3:7571518626288998598:2048]
2025-11-11T17:19:15.366040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518626288998598:2048] [node 1] ICS09 handshake done sender: [3:7571518626288998582:2117] self: [3:7571518626288998596:0] peer: [1:7571518626229980510:0] socket: 285 qp: -1
2025-11-11T17:19:15.366080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518626288998598:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:15.366113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:15.384551Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518626288998584:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.384586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518626288998584:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.403500Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518626288998580:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.403536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518626288998580:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.405813Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518626288998584:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:15.435192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518621994031268:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.476475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:15.477887Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518626288998580:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:15.481738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518621994031268:2061] [node 6] ICP22 created new session: [3:7571518626288998602:2061]
2025-11-11T17:19:15.313944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518626229980527:2048] [node 2] ICS09 handshake done sender: [1:7571518626229980512:2116] self: [1:7571518626229980509:0] peer: [2:7571518629664131450:0] socket: 289 qp: -1
2025-11-11T17:19:15.314000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518626229980527:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:15.314063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:15.314559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518621935013134:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(68537683ns) {HandleHandshakeStatus:329(68535613ns) {DropIncomingHandshake:441(14253ns) DropOutgoingHandshake:456(289ns) SwitchToState:178(536ns) ProcessPendingSessionEvents:495(104559ns) {ForwardSessionEventToSession:612(58526ns) ForwardSessionEventToSession:612(7378ns) ForwardSessionEventToSession:612(2447ns) ForwardSessionEventToSession:612(2105ns) ForwardSessionEventToSession:612(4876ns) ForwardSessionEventToSession:612(5443ns) ForwardSessionEventToSession:612(2671ns) ForwardSessionEventToSession:612(3796ns) ForwardSessionEventToSession:612(2870ns) ForwardSessionEventToSession:612(5259ns)}}}
2025-11-11T17:19:15.336777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518621935013135:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.353394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518621935013197:2079] [node 5] ICP35 event processing took too much time PendingConnection:278(127079042ns) {HandleHandshakeStatus:329(127075545ns) {DropIncomingHandshake:441(1894ns) DropOutgoingHandshake:456(604ns) SwitchToState:178(484ns) ProcessPendingSessionEvents:495(19428427ns) {ForwardSessionEventToSession:612(19370735ns) ForwardSessionEventToSession:612(8879ns) ForwardSessionEventToSession:612(5769ns) ForwardSessionEventToSession:612(2237ns) ForwardSessionEventToSession:612(4788ns) ForwardSessionEventToSession:612(1948ns) ForwardSessionEventToSession:612(2137ns) ForwardSessionEventToSession:612(4230ns) ForwardSessionEventToSession:612(3229ns) ForwardSessionEventToSession:612(9107ns)}}}
2025-11-11T17:19:15.353454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518621935013197:2079] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:15.385953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:15.402088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518621935013135:2049] [node 3] ICP22 created new session: [1:7571518626229980532:2049]
2025-11-11T17:19:15.404050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518626229980532:2049] [node 3] ICS09 handshake done sender: [1:7571518626229980511:2115] self: [1:7571518626229980510:0] peer: [3:7571518626288998596:0] socket: 284 qp: -1
2025-11-11T17:19:15.407854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518626229980532:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:15.414179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:15.455066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518621935013135:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(118097621ns) {HandleHandshakeStatus:329(118095887ns) {DropIncomingHandshake:441(16491ns) DropOutgoingHandshake:456(3819ns) SwitchToState:178(579ns) ProcessPendingSessionEvents:495(17194422ns) {ForwardSessionEventToSession:612(17142006ns) ForwardSessionEventToSession:612(4899ns) ForwardSessionEventToSession:612(6504ns) ForwardSessionEventToSession:612(3146ns) ForwardSessionEventToSession:612(3029ns) ForwardSessionEventToSession:612(7783ns) ForwardSessionEventToSession:612(5370ns) ForwardSessionEventToSession:612(6484ns) ForwardSessionEventToSession:612(2712ns) ForwardSessionEventToSession:612(2810ns)}}}
2025-11-11T17:19:15.457815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518623939814590:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.469715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:15.357104Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518629664131448:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.492127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518621994031252:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.492184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:15.494059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518626288998602:2061] [node 6] ICS09 handshake done sender: [3:7571518626288998584:2118] self: [3:7571518626288998583:0] peer: [6:7571518628234781985:0] socket: 302 qp: -1
2025-11-11T17:19:15.501875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518626288998602:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:15.505669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518621994031252:2062] [node 5] ICP22 created new session: [3:7571518626288998603:2062]
2025-11-11T17:19:15.508465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:15.509657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518626288998603:2062] [node 5] ICS09 handshake done sender: [3:7571518626288998580:2116] self: [3:7571518626288998577:0] peer: [5:7571518628700254547:0] socket: 274 qp: -1
2025-11-11T17:19:15.357159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518629664131448:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:15.457865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:15.521719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518626288998603:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:15.521800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:15.522959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518621994031268:2061] [node 6] ICP35 event processing took too much time PendingConnection:278(94066464ns) {HandleHandshakeStatus:329(94063836ns) {DropIncomingHandshake:441(18822ns) DropOutgoingHandshake:456(224ns) SwitchToState:178(4206ns) ProcessPendingSessionEvents:495(579443ns) {ForwardSessionEventToSession:612(526446ns) ForwardSessionEventToSession:612(4719ns) ForwardSessionEventToSession:612(10679ns) ForwardSessionEventToSession:612(6164ns) ForwardSessionEventToSession:612(2382ns) ForwardSessionEventToSession:612(6143ns) ForwardSessionEventToSession:612(3333ns) ForwardSessionEventToSession:612(2630ns) ForwardSessionEventToSession:612(6036ns) ForwardSessionEventToSession:612(1972ns)}}}
2025-11-11T17:19:15.523678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518621994031252:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(31367529ns) {HandleHandshakeStatus:329(31365438ns) {DropIncomingHandshake:441(12928ns) DropOutgoingHandshake:456(304ns) SwitchToState:178(529ns) ProcessPendingSessionEvents:495(1552971ns) {ForwardSessionEventToSession:612(377825ns) ForwardSessionEventToSession:612(1132400ns) ForwardSessionEventToSession:612(3221ns) ForwardSessionEventToSession:612(2771ns) ForwardSessionEventToSession:612(5127ns) ForwardSessionEventToSession:612(2563ns) ForwardSessionEventToSession:612(3071ns) ForwardSessionEventToSession:612(7748ns) ForwardSessionEventToSession:612(3586ns) ForwardSessionEventToSession:612(5041ns)}}}
2025-11-11T17:19:15.361905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518625369164061:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518629664131444:2117] held: no
2025-11-11T17:19:15.369211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518625369164131:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.369288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:15.465737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518623939814590:2050] [node 3] ICP22 created new session: [6:7571518628234781987:2050]
2025-11-11T17:19:15.369336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518625369164131:2064] [node 5] ICP22 created new session: [2:7571518629664131454:2064]
2025-11-11T17:19:15.369378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518629664131454:2064] [node 5] ICS09 handshake done sender: [2:7571518629664131439:2115] self: [2:7571518629664131438:0] peer: [5:7571518628700254542:0] socket: 277 qp: -1
2025-11-11T17:19:15.369411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518629664131454:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:15.465796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518628234781987:2050] [node 3] ICS09 handshake done sender: [6:7571518628234781983:2123] self: [6:7571518628234781985:0] peer: [3:7571518626288998583:0] socket: 244 qp: -1
2025-11-11T17:19:15.369446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:15.469676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518628234781987:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:15.371398Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518629664131448:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:15.469730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:15.373852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518629664131451:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2087
2025-11-11T17:19:15.374030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518629664131452:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:15.490223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518623939814590:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(32195001ns) {HandleHandshakeStatus:329(32192681ns) {DropIncomingHandshake:441(495ns) DropOutgoingHandshake:456(10027ns) SwitchToState:178(389ns) ProcessPendingSessionEvents:495(13798311ns) {ForwardSessionEventToSession:612(13750251ns) ForwardSessionEventToSession:612(4344ns) ForwardSessionEventToSession:612(6657ns) ForwardSessionEventToSession:612(5247ns) ForwardSessionEventToSession:612(2239ns) ForwardSessionEventToSession:612(2427ns) ForwardSessionEventToSession:612(6505ns) ForwardSessionEventToSession:612(2504ns) ForwardSessionEventToSession:612(4587ns) ForwardSessionEventToSession:612(2244ns)}}}
2025-11-11T17:19:15.485722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518624405287207:2064] [node 4] ICP22 created new session: [5:7571518628700254551:2064]
2025-11-11T17:19:15.486222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518628700254552:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46079
2025-11-11T17:19:15.488070Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518628700254543:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:15.488254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518624405287158:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.488319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:15.488358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518624405287158:2050] [node 3] ICP22 created new session: [5:7571518628700254553:2050]
2025-11-11T17:19:15.488396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518628700254553:2050] [node 3] ICS09 handshake done sender: [5:7571518628700254543:2119] self: [5:7571518628700254547:0] peer: [3:7571518626288998577:0] socket: 303 qp: -1
2025-11-11T17:19:15.488429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518628700254553:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:15.488460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:15.508503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518628700254551:2064] [node 4] ICS09 handshake done sender: [5:7571518628700254544:2120] self: [5:7571518628700254546:0] peer: [4:7571518625999799659:0] socket: 276 qp: -1
2025-11-11T17:19:15.513715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518628700254551:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:15.513788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:15.514027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518624405287207:2064] [node 4] ICP35 event processing took too much time PendingConnection:278(56166931ns) {HandleHandshakeStatus:329(56164722ns) {DropIncomingHandshake:441(765ns) DropOutgoingHandshake:456(524ns) SwitchToState:178(1103ns) ProcessPendingSessionEvents:495(1121ns)}}
2025-11-11T17:19:15.378141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518625369164134:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:15.398882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:15.403627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518625369164134:2065] [node 4] ICP22 created new session: [2:7571518629664131457:2065]
2025-11-11T17:19:15.421640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518629664131457:2065] [node 4] ICS09 handshake done sender: [2:7571518629664131448:2118] self: [2:7571518629664131447:0] peer: [4:7571518625999799669:0] socket: 292 qp: -1
2025-11-11T17:19:15.421889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518625369164061:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(79082171ns) {HandleHandshakeStatus:395(79079721ns) {LogHandshakeFail:472(14231ns) {UpdateErrorStateLog:544(4242ns)} DropOutgoingHandshake:456(1533ns) ProcessPendingSessionEvents:495(19868903ns) {DropSessionEvent:512(14792ns) DropSessionEvent:512(4402ns) DropSessionEvent:512(3663ns) DropSessionEvent:512(4453ns) DropSessionEvent:512(4291ns) DropSessionEvent:512(11911512ns) DropSessionEvent:512(5603ns) DropSessionEvent:512(7902029ns) DropSessionEvent:512(5032ns) DropSessionEvent:512(4339ns)}}}
2025-11-11T17:19:15.421979Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518629664131444:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:15.422490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518629664131457:2065] [node 4] ICS10 traffic start
2025-11-11T17:19:15.422532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:15.434189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518625369164061:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:15.474689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518625369164134:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(100799319ns) {HandleHandshakeStatus:329(100797131ns) {DropIncomingHandshake:441(813ns) DropOutgoingHandshake:456(2231ns) SwitchToState:178(577ns) ProcessPendingSessionEvents:495(18749060ns) {ForwardSessionEventToSession:612(18691083ns) ForwardSessionEventToSession:612(6544ns) ForwardSessionEventToSession:612(5084ns) ForwardSessionEventToSession:612(3789ns) ForwardSessionEventToSession:612(9484ns) ForwardSessionEventToSession:612(7285ns) ForwardSessionEventToSession:612(3233ns) ForwardSessionEventToSession:612(5149ns) ForwardSessionEventToSession:612(2169ns) ForwardSessionEventToSession:612(2082ns)}}}
2025-11-11T17:19:15.474765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518625369164134:2065] [node 4] ICP18 handshake (actor [2:7571518629664131452:2120]) is too old
2025-11-11T17:19:15.475840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:15.475901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518625369164061:2048] [node 1] ICP22 created new session: [2:7571518629664131459:2048]
2025-11-11T17:19:15.475941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518629664131459:2048] [node 1] ICS09 handshake done sender: [2:7571518629664131444:2117] self: [2:7571518629664131450:0] peer: [1:7571518626229980509:0] socket: 290 qp: -1
2025-11-11T17:19:15.475977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518629664131459:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:15.476034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:15.476212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518625369164061:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(41960465ns) {HandleHandshakeStatus:329(41958763ns) {DropIncomingHandshake:441(1023ns) DropOutgoingHandshake:456(288ns) SwitchToState:178(515ns) ProcessPendingSessionEvents:495(543ns)}}
2025-11-11T17:19:15.592714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:15.592950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518629664131459:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-11T17:19:15.592988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518625369164061:2048] [node 1] ICP30 unregister session Session# [2:7571518629664131459:2048] VirtualId# [2:7571518629664131450:0]
2025-11-11T17:19:15.593025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518629664131459:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.593104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:15.593141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:15.595463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:15.595805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518626288998598:2048] [node 1] ICS01 socket: 285 reason# EndOfStream
2025-11-11T17:19:15.595858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518621994031204:2048] [node 1] ICP30 unregister session Session# [3:7571518626288998598:2048] VirtualId# [3:7571518626288998596:0]
2025-11-11T17:19:15.595888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518626288998598:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.595970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:15.596026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:15.596490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:15.605698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:15.606359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:15.606582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518628700254539:2048] [node 1] ICS01 socket: 260 reason# EndOfStream
2025-11-11T17:19:15.606614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518624405287156:2048] [node 1] ICP30 unregister session Session# [5:7571518628700254539:2048] VirtualId# [5:7571518628700254533:0]
2025-11-11T17:19:15.606642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518628700254539:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.606705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:15.606740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:15.605953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518625999799666:2048] [node 1] ICS07 socket disconnect 250 reason# EndOfStream
2025-11-11T17:19:15.605982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518625999799666:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.606060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:15.606098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518625999799666:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:15.605862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518628234781977:2048] [node 1] ICS01 socket: 247 reason# EndOfStream
2025-11-11T17:19:15.625993Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Couldn't connect to any resolved address
2025-11-11T17:19:15.626322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518621704832277:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:15.626373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518625999799666:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:15.626410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518621704832277:2048] [node 1] ICP30 unregister session Session# [4:7571518625999799666:2048] VirtualId# [4:7571518625999799650:0]
2025-11-11T17:19:15.626458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:15.626486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518621704832277:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:15.626499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:15.605927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518623939814588:2048] [node 1] ICP30 unregister session Session# [6:7571518628234781977:2048] VirtualId# [6:7571518628234781965:0]
2025-11-11T17:19:15.605967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518628234781977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.606060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:15.606105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:15.633672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518621704832277:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:15.705014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:15.705223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518626288998591:2049] [node 2] ICS01 socket: 264 reason# EndOfStream
2025-11-11T17:19:15.705268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518621994031205:2049] [node 2] ICP30 unregister session Session# [3:7571518626288998591:2049] VirtualId# [3:7571518626288998585:0]
2025-11-11T17:19:15.705307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518626288998591:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.705409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:15.705456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:15.709767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:15.710297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:15.710483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518628234781972:2049] [node 2] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-11T17:19:15.709985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518628700254549:2049] [node 2] ICS07 socket disconnect 252 reason# EndOfStream
2025-11-11T17:19:15.710507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518628234781972:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.710021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518628700254549:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.710569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:15.710094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:15.710132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518628700254549:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:15.710601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518628234781972:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:15.710587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:15.710822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518625999799676:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-11-11T17:19:15.710863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518621704832278:2049] [node 2] ICP30 unregister session Session# [4:7571518625999799676:2049] VirtualId# [4:7571518625999799669:0]
2025-11-11T17:19:15.710893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518625999799676:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.710942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:15.710979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:15.711621Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address
2025-11-11T17:19:15.711674Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address
2025-11-11T17:19:15.712064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518623939814589:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:15.711910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518624405287157:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:15.712418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518628234781972:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:15.712456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518623939814589:2049] [node 2] ICP30 unregister session Session# [6:7571518628234781972:2049] VirtualId# [6:7571518623939814667:0]
2025-11-11T17:19:15.712485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:15.712513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518623939814589:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:15.712528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:15.711948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518628700254549:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:15.712119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518624405287157:2049] [node 2] ICP30 unregister session Session# [5:7571518628700254549:2049] VirtualId# [5:7571518628700254542:0]
2025-11-11T17:19:15.712161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:15.712192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518624405287157:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:15.712210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9992) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:15.725505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518624405287157:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:15.725611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518623939814589:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:15.775275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:15.775496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518628234781987:2050] [node 3] ICS01 socket: 244 reason# EndOfStream
2025-11-11T17:19:15.775543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518623939814590:2050] [node 3] ICP30 unregister session Session# [6:7571518628234781987:2050] VirtualId# [6:7571518628234781985:0]
2025-11-11T17:19:15.775583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518628234781987:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.775662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:15.775734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:15.779810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:15.780020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518625999799665:2050] [node 3] ICS01 socket: 266 reason# EndOfStream
2025-11-11T17:19:15.780058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518621704832279:2050] [node 3] ICP30 unregister session Session# [4:7571518625999799665:2050] VirtualId# [4:7571518625999799654:0]
2025-11-11T17:19:15.780095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518625999799665:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.780173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:15.780215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:15.781348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:15.781534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518628700254553:2050] [node 3] ICS01 socket: 303 reason# EndOfStream
2025-11-11T17:19:15.781575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518624405287158:2050] [node 3] ICP30 unregister session Session# [5:7571518628700254553:2050] VirtualId# [5:7571518628700254547:0]
2025-11-11T17:19:15.781616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518628700254553:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.781690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:15.781729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:15.855042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:15.855291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518628700254551:2064] [node 4] ICS01 socket: 276 reason# EndOfStream
2025-11-11T17:19:15.855332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518624405287207:2064] [node 4] ICP30 unregister session Session# [5:7571518628700254551:2064] VirtualId# [5:7571518628700254546:0]
2025-11-11T17:19:15.855366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518628700254551:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.855459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:15.855502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:15.855587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:15.855776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518628234781981:2059] [node 4] ICS01 socket: 261 reason# EndOfStream
2025-11-11T17:19:15.855803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518623939814601:2059] [node 4] ICP30 unregister session Session# [6:7571518628234781981:2059] VirtualId# [6:7571518628234781971:0]
2025-11-11T17:19:15.855823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518628234781981:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.855873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:15.855907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:15.973789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:15.974151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518628234781978:2065] [node 5] ICS01 socket: 256 reason# EndOfStream
2025-11-11T17:19:15.974225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518623939814610:2065] [node 5] ICP30 unregister session Session# [6:7571518628234781978:2065] VirtualId# [6:7571518628234781966:0]
2025-11-11T17:19:15.974291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518628234781978:2065] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:15.974437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:15.974520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:16.173900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518631331264234:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:16.173948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518631331264235:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:16.221864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518633190985283:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:16.221902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518633190985284:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:16.281894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518629790195227:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:16.281939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518629790195228:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:16.315095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518630090039599:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:16.315130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518630090039600:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:16.315154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518630090039601:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:16.405825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518632006689584:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:16.405875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518632006689585:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:16.405898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518632006689586:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:16.638907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518633282129507:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:16.638979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518633282129508:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:16.639016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518633282129509:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:16.639593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518633282129525:2064] [node 5] ICP01 ready to work
2025-11-11T17:19:16.625140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518632006689603:2060] [node 4] ICP01 ready to work
2025-11-11T17:19:16.679332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518632006689603:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53867413ns) {Configure:104(53863115ns) {ProcessConfigured:135(74911ns) {StartInitialHandshake:157(71744ns) {DropHandshakes:469(6505ns) {DropIncomingHandshake:441(951ns) DropOutgoingHandshake:456(559ns)} GenerateSessionVirtualId:988(966ns) PrepareNewSessionHandshake:476(1765ns) {SwitchToState:178(705ns)}}}}}
2025-11-11T17:19:16.741172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518630090039617:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:16.790087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518633190985346:2086] [node 4] ICP01 ready to work
2025-11-11T17:19:16.773555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518629790195278:2060] [node 5] ICP01 ready to work
2025-11-11T17:19:16.845871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518629790195290:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:16.838720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518632006689586:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(159159750ns) {Configure:104(159155140ns) {ProcessConfigured:135(28735524ns) {StartInitialHandshake:157(28731604ns) {DropHandshakes:469(2321ns) {DropIncomingHandshake:441(506ns) DropOutgoingHandshake:456(335ns)} GenerateSessionVirtualId:988(1117ns) PrepareNewSessionHandshake:476(4190ns) {SwitchToState:178(2395ns)}}}}}
2025-11-11T17:19:16.790395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518633190985347:2088] [node 5] ICP01 ready to work
2025-11-11T17:19:16.833688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518630090039653:2064] [node 5] ICP01 ready to work
2025-11-11T17:19:16.913833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518629790195292:2068] [node 4] ICP01 ready to work
2025-11-11T17:19:16.790660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518633190985348:2091] [node 6] ICP01 ready to work
2025-11-11T17:19:16.874120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518633282129508:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(232650509ns) {Configure:104(232646906ns) {ProcessConfigured:135(8020929ns) {StartInitialHandshake:157(8017925ns) {DropHandshakes:469(2404ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(383ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:476(4023ns) {SwitchToState:178(2096ns)}}}}}
2025-11-11T17:19:16.888882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518633282129567:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:16.952154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518633190985362:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18727
2025-11-11T17:19:16.956593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518631331264235:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(320175981ns) {Configure:104(320168108ns) {ProcessConfigured:135(7475814ns) {StartInitialHandshake:157(7470993ns) {DropHandshakes:469(1114ns) {DropIncomingHandshake:441(156ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(668ns) PrepareNewSessionHandshake:476(3349ns) {SwitchToState:178(605ns)}}}}}
2025-11-11T17:19:16.957014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518631331264299:2087] [node 5] ICP01 ready to work
2025-11-11T17:19:16.957340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518631331264300:2089] [node 4] ICP01 ready to work
2025-11-11T17:19:16.933861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518629790195290:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16896772ns) {Configure:104(16893521ns) {ProcessConfigured:135(6428084ns) {StartInitialHandshake:157(6426754ns) {DropHandshakes:469(1901ns) {DropIncomingHandshake:441(261ns) DropOutgoingHandshake:456(312ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:476(2928ns) {SwitchToState:178(430ns)}}}}}
2025-11-11T17:19:16.934988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518633282129567:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45536779ns) {Configure:104(45533235ns) {ProcessConfigured:135(63065ns) {StartInitialHandshake:157(61613ns) {DropHandshakes:469(1339ns) {DropIncomingHandshake:441(370ns) DropOutgoingHandshake:456(259ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:476(4249ns) {SwitchToState:178(678ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:16.935380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518632006689658:2084] [node 6] ICP01 ready to work
2025-11-11T17:19:16.960498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518630090039600:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(33908679ns) {Configure:104(33904205ns) {ProcessConfigured:135(386964ns) {StartInitialHandshake:157(383184ns) {DropHandshakes:469(1830ns) {DropIncomingHandshake:441(422ns) DropOutgoingHandshake:456(299ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:476(6176ns) {SwitchToState:178(546ns)}}}}}
2025-11-11T17:19:16.950800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518629790195227:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16703424ns) {Configure:104(16695648ns) {ProcessConfigured:135(48371ns) {StartInitialHandshake:157(46801ns) {DropHandshakes:469(2257ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(347ns)} GenerateSessionVirtualId:988(516ns) PrepareNewSessionHandshake:476(1063ns) {SwitchToState:178(453ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:16.974949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518633190985350:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:16.974980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518633190985350:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:16.978277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518632006689662:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:16.962957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518629790195303:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59221
2025-11-11T17:19:16.981748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518631331264301:2093] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:16.995520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518633282129585:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:16.975843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518633190985363:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18727
2025-11-11T17:19:16.963013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518629790195304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59221
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.013170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518630090039599:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(42903553ns) {Configure:104(42899801ns) {ProcessConfigured:135(62545ns) {StartInitialHandshake:157(60809ns) {DropHandshakes:469(1180ns) {DropIncomingHandshake:441(334ns) DropOutgoingHandshake:456(236ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:476(5047ns) {SwitchToState:178(695ns)}}}}}
2025-11-11T17:19:17.013952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518630090039668:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6954
2025-11-11T17:19:17.014063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518630090039669:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6954
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.018189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656959:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
2025-11-11T17:19:17.018240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656960:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.026048Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518636301656959:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.026164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518632006689585:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.027317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656962:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
2025-11-11T17:19:17.027890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518636301656959:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:17.028035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518632006689585:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.028078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:17.028116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518632006689585:2049] [node 2] ICP22 created new session: [5:7571518636301656963:2049]
2025-11-11T17:19:17.028154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518636301656963:2049] [node 2] ICS09 handshake done sender: [5:7571518636301656959:2116] self: [5:7571518636301656961:0] peer: [2:7571518633190985355:0] socket: 276 qp: -1
2025-11-11T17:19:17.028180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518636301656963:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:17.028212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:17.035234Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518630090039668:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.035319Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518630090039669:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.036769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518630090039600:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:16.977225Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518633190985350:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:16.963158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518629790195305:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59221
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.044974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096882:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:17.045064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096883:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:16.974158Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518629790195304:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:16.974316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518629790195228:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:16.984606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518629790195310:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59221
2025-11-11T17:19:17.053734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518631331264301:2093] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18530518ns) {Configure:104(18524777ns) {ProcessConfigured:135(17297539ns) {StartInitialHandshake:157(17295749ns) {DropHandshakes:469(2293ns) {DropIncomingHandshake:441(391ns) DropOutgoingHandshake:456(317ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:476(3788ns) {SwitchToState:178(1688ns)}}}}}
2025-11-11T17:19:17.061072Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518633282129585:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.061178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518633282129509:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.058712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518629790195303:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.059039Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518629790195304:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:17.059645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518629790195305:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.059678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518629790195228:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.059730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:17.059775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518629790195228:2049] [node 2] ICP22 created new session: [3:7571518634085162607:2049]
2025-11-11T17:19:17.059818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518634085162607:2049] [node 2] ICS09 handshake done sender: [3:7571518629790195304:2116] self: [3:7571518629790195309:0] peer: [2:7571518633190985349:0] socket: 257 qp: -1
2025-11-11T17:19:17.059849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518634085162607:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:17.059880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.067201Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518632006689662:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.067333Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518636301656960:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.068234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518632006689586:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.068294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518632006689603:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.070662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656968:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
2025-11-11T17:19:17.071031Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518636301656960:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:17.068714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518630090039671:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.068740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518630090039671:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.069437Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518630090039671:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:17.069695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518630090039653:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.069738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:17.069824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518630090039653:2064] [node 5] ICP22 created new session: [4:7571518634385006978:2064]
2025-11-11T17:19:17.069863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518634385006978:2064] [node 5] ICS09 handshake done sender: [4:7571518630090039671:2114] self: [4:7571518630090039670:0] peer: [5:7571518636301656967:0] socket: 275 qp: -1
2025-11-11T17:19:17.069891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518634385006978:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:17.069924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:16.977536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518633190985284:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.066735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518635626231598:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10566
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.071681Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10779) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:17.078087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518637577096882:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.072644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518632006689603:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.087283Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518637577096883:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.072687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:17.090660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518633282129567:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.072721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518632006689603:2060] [node 4] ICP22 created new session: [5:7571518636301656969:2060]
2025-11-11T17:19:17.090745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096885:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:17.072753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518636301656969:2060] [node 4] ICS09 handshake done sender: [5:7571518636301656960:2117] self: [5:7571518636301656967:0] peer: [4:7571518630090039670:0] socket: 277 qp: -1
2025-11-11T17:19:17.072778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518636301656969:2060] [node 4] ICS10 traffic start
2025-11-11T17:19:17.072803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:17.072963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518632006689603:2060] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:16.977576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.065293Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518629790195298:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.066854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518635626231599:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10566
2025-11-11T17:19:17.066948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518635626231600:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10566
2025-11-11T17:19:17.078005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231600:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.089938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231599:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.090242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518635626231610:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10566
2025-11-11T17:19:17.065324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518629790195298:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:16.977632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518633190985284:2049] [node 3] ICP22 created new session: [2:7571518633190985365:2049]
2025-11-11T17:19:16.977667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518633190985365:2049] [node 3] ICS09 handshake done sender: [2:7571518633190985350:2110] self: [2:7571518633190985349:0] peer: [3:7571518629790195309:0] socket: 248 qp: -1
2025-11-11T17:19:16.987159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518633190985365:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:16.987209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:17.012040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518633190985284:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(34346302ns) {HandleHandshakeStatus:329(34343937ns) {DropIncomingHandshake:441(350ns) DropOutgoingHandshake:456(1145ns) SwitchToState:178(183ns) ProcessPendingSessionEvents:495(2475269ns) {ForwardSessionEventToSession:612(2421096ns) ForwardSessionEventToSession:612(9597ns) ForwardSessionEventToSession:612(3091ns) ForwardSessionEventToSession:612(4788ns) ForwardSessionEventToSession:612(8172ns) ForwardSessionEventToSession:612(1954ns) ForwardSessionEventToSession:612(5210ns) ForwardSessionEventToSession:612(2237ns) ForwardSessionEventToSession:612(5249ns) ForwardSessionEventToSession:612(2109ns)}}}
2025-11-11T17:19:17.023328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518637485952664:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18727
2025-11-11T17:19:17.026362Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518633190985356:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.026382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518633190985356:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.027460Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518633190985356:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:17.062354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518633190985347:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.062389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:17.062436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518633190985347:2088] [node 5] ICP22 created new session: [2:7571518637485952666:2088]
2025-11-11T17:19:17.062474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518637485952666:2088] [node 5] ICS09 handshake done sender: [2:7571518633190985356:2113] self: [2:7571518633190985355:0] peer: [5:7571518636301656961:0] socket: 272 qp: -1
2025-11-11T17:19:17.062502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518637485952666:2088] [node 5] ICS10 traffic start
2025-11-11T17:19:17.062533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:17.077909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518633190985362:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.091153Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518633190985354:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.091184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518633190985354:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.094091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656971:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
2025-11-11T17:19:17.081816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518629790195298:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:17.113962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518629790195294:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.113986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518629790195294:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.114543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518629790195290:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.114600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:17.114643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518629790195290:2063] [node 6] ICP22 created new session: [3:7571518634085162615:2063]
2025-11-11T17:19:17.114683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518634085162615:2063] [node 6] ICS09 handshake done sender: [3:7571518629790195298:2112] self: [3:7571518629790195297:0] peer: [6:7571518637577096884:0] socket: 265 qp: -1
2025-11-11T17:19:17.114715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518634085162615:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:17.114774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:17.117790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518633282129525:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.118115Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518633282129585:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:17.118429Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518630090039667:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.118460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518630090039667:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.118387Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518632006689661:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.118414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518632006689661:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.119980Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.120736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518633282129509:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.120807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:17.120848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518633282129509:2050] [node 3] ICP22 created new session: [6:7571518637577096890:2050]
2025-11-11T17:19:17.120891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518637577096890:2050] [node 3] ICS09 handshake done sender: [6:7571518633282129585:2115] self: [6:7571518637577096884:0] peer: [3:7571518629790195297:0] socket: 266 qp: -1
2025-11-11T17:19:17.120918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518637577096890:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:17.120951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:17.121440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096888:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.124348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518634385006981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6954
2025-11-11T17:19:17.122289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518633190985284:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:17.122910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518637485952664:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.124673Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518632006689661:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:17.124842Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518630090039667:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.125116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518630090039617:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.125154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:17.125192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518630090039617:2063] [node 6] ICP22 created new session: [4:7571518634385006983:2063]
2025-11-11T17:19:17.125233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518634385006983:2063] [node 6] ICS09 handshake done sender: [4:7571518630090039667:2111] self: [4:7571518630090039666:0] peer: [6:7571518637577096886:0] socket: 274 qp: -1
2025-11-11T17:19:17.125261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518634385006983:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:17.125289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:17.126770Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518636301656971:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.126960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518632006689658:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.126990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:17.127026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518632006689658:2084] [node 6] ICP22 created new session: [5:7571518636301656973:2084]
2025-11-11T17:19:17.127058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518636301656973:2084] [node 6] ICS09 handshake done sender: [5:7571518632006689661:2114] self: [5:7571518632006689660:0] peer: [6:7571518637577096887:0] socket: 281 qp: -1
2025-11-11T17:19:17.127082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518636301656973:2084] [node 6] ICS10 traffic start
2025-11-11T17:19:17.127108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:17.127556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518632006689584:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.126019Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231598:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.126321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518635626231615:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10566
2025-11-11T17:19:17.128059Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231604:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.128078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518635626231604:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.128982Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518635626231604:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:17.129274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518631331264299:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.129353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:17.129396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518631331264299:2087] [node 5] ICP22 created new session: [1:7571518635626231618:2087]
2025-11-11T17:19:17.129433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518635626231618:2087] [node 5] ICS09 handshake done sender: [1:7571518635626231604:2115] self: [1:7571518635626231603:0] peer: [5:7571518636301656975:0] socket: 290 qp: -1
2025-11-11T17:19:17.129458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518635626231618:2087] [node 5] ICS10 traffic start
2025-11-11T17:19:17.129482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.133018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518634085162613:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59221
2025-11-11T17:19:17.133133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518634085162614:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59221
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.135197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518634385006986:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6954
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.136037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656976:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
2025-11-11T17:19:17.136297Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518633190985354:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:17.137279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518633190985346:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.137323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:17.137359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518633190985346:2086] [node 4] ICP22 created new session: [2:7571518637485952673:2086]
2025-11-11T17:19:17.137393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518637485952673:2086] [node 4] ICS09 handshake done sender: [2:7571518633190985354:2112] self: [2:7571518633190985353:0] peer: [4:7571518634385006976:0] socket: 250 qp: -1
2025-11-11T17:19:17.137418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518637485952673:2086] [node 4] ICS10 traffic start
2025-11-11T17:19:17.137441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.138101Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518637577096888:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.138332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096889:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.138764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656977:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
2025-11-11T17:19:17.138837Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:17.141558Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518637577096883:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.143153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518637485952672:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18727
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.143490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518633282129507:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.143565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:17.143657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096893:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:17.144137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518632006689584:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518636301656971:2120] held: no
2025-11-11T17:19:17.144242Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518636301656971:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:17.144725Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:17.145598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518633282129525:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.145667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:17.145706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518633282129525:2064] [node 5] ICP22 created new session: [6:7571518637577096896:2064]
2025-11-11T17:19:17.145744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518637577096896:2064] [node 5] ICS09 handshake done sender: [6:7571518637577096883:2117] self: [6:7571518637577096887:0] peer: [5:7571518632006689660:0] socket: 283 qp: -1
2025-11-11T17:19:17.145772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518637577096896:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:17.145814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.147161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518632006689584:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.147197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:17.147232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518632006689584:2048] [node 1] ICP22 created new session: [5:7571518636301656979:2048]
2025-11-11T17:19:17.147271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518636301656979:2048] [node 1] ICS09 handshake done sender: [5:7571518636301656971:2120] self: [5:7571518636301656975:0] peer: [1:7571518635626231603:0] socket: 291 qp: -1
2025-11-11T17:19:17.147294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518636301656979:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:17.147320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:17.147818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518636301656978:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53792
2025-11-11T17:19:17.148255Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231615:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.148346Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231602:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.148363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518635626231602:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.149513Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518629790195294:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:17.149924Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518634085162613:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.150085Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518634085162614:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.150118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518629790195278:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.150159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:17.150193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518629790195278:2060] [node 5] ICP22 created new session: [3:7571518634085162619:2060]
2025-11-11T17:19:17.150227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518634085162619:2060] [node 5] ICS09 handshake done sender: [3:7571518629790195294:2110] self: [3:7571518629790195293:0] peer: [5:7571518636301656966:0] socket: 259 qp: -1
2025-11-11T17:19:17.150249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518634085162619:2060] [node 5] ICS10 traffic start
2025-11-11T17:19:17.150280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:17.151175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518630090039600:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518630090039669:2113] held: no
2025-11-11T17:19:17.151285Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518630090039669:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:17.151599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518630090039600:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.151637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:17.151669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518630090039600:2049] [node 2] ICP22 created new session: [4:7571518634385006987:2049]
2025-11-11T17:19:17.151715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518634385006987:2049] [node 2] ICS09 handshake done sender: [4:7571518630090039669:2113] self: [4:7571518634385006976:0] peer: [2:7571518633190985353:0] socket: 262 qp: -1
2025-11-11T17:19:17.151740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518634385006987:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:17.151771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:17.152520Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518637485952672:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.152703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518633190985283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.153166Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518635626231602:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:17.153503Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518632006689662:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:17.153714Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518637577096882:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:17.155231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518629790195227:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.155713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231610:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.155801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231606:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.155819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518635626231606:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.156338Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518631331264298:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.156354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518631331264298:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.156934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518631331264301:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.156999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:17.157035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518631331264301:2093] [node 6] ICP22 created new session: [1:7571518635626231623:2093]
2025-11-11T17:19:17.157070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518635626231623:2093] [node 6] ICS09 handshake done sender: [1:7571518635626231602:2114] self: [1:7571518635626231601:0] peer: [6:7571518637577096895:0] socket: 289 qp: -1
2025-11-11T17:19:17.157099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518635626231623:2093] [node 6] ICS10 traffic start
2025-11-11T17:19:17.157129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.158311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518637577096892:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.158353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518633282129567:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.158417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:17.158452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518633282129567:2068] [node 4] ICP22 created new session: [6:7571518637577096899:2068]
2025-11-11T17:19:17.158488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518637577096899:2068] [node 4] ICS09 handshake done sender: [6:7571518637577096882:2116] self: [6:7571518637577096886:0] peer: [4:7571518630090039666:0] socket: 282 qp: -1
2025-11-11T17:19:17.158508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518637577096899:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:17.158531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:17.159308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096898:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:17.159747Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:17.160450Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:17.161388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518632006689586:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.161430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:17.161466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518632006689586:2050] [node 3] ICP22 created new session: [5:7571518636301656981:2050]
2025-11-11T17:19:17.161505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518636301656981:2050] [node 3] ICS09 handshake done sender: [5:7571518632006689662:2115] self: [5:7571518636301656966:0] peer: [3:7571518629790195293:0] socket: 263 qp: -1
2025-11-11T17:19:17.161532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518636301656981:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:17.161554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:17.162006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518633282129508:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.162205Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518637577096888:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:17.162412Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518633190985358:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.162438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518633190985358:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.163422Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518633190985358:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:17.163699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518633190985348:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.163744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:17.163773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518633190985348:2091] [node 6] ICP22 created new session: [2:7571518637485952677:2091]
2025-11-11T17:19:17.163808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518637485952677:2091] [node 6] ICS09 handshake done sender: [2:7571518633190985358:2114] self: [2:7571518633190985357:0] peer: [6:7571518637577096902:0] socket: 297 qp: -1
2025-11-11T17:19:17.163831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518637485952677:2091] [node 6] ICS10 traffic start
2025-11-11T17:19:17.163854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:17.164638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518634385006986:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.164728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518630090039601:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.164963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518629790195302:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.164985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518629790195302:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:17.165866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518632006689586:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:17.166438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518633282129507:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.166492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:17.166528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518633282129507:2048] [node 1] ICP22 created new session: [6:7571518637577096904:2048]
2025-11-11T17:19:17.166559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518637577096904:2048] [node 1] ICS09 handshake done sender: [6:7571518637577096888:2119] self: [6:7571518637577096895:0] peer: [1:7571518635626231601:0] socket: 294 qp: -1
2025-11-11T17:19:17.166585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518637577096904:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:17.166609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.169122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518634385006990:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6954
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.169378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518637485952679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.169914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518634085162624:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59221
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.170237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518634385006991:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6954
2025-11-11T17:19:17.173766Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518629790195302:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:17.174213Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518634385006986:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:17.174406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518630090039601:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.174460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:17.174506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518630090039601:2050] [node 3] ICP22 created new session: [4:7571518634385006992:2050]
2025-11-11T17:19:17.174539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518634385006992:2050] [node 3] ICS09 handshake done sender: [4:7571518634385006986:2118] self: [4:7571518634385006989:0] peer: [3:7571518629790195301:0] socket: 304 qp: -1
2025-11-11T17:19:17.174566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518634385006992:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:17.174597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:17.176034Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518635626231606:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:17.176482Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518637485952672:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:17.176687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518633190985283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.176732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:17.176761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518633190985283:2048] [node 1] ICP22 created new session: [2:7571518637485952680:2048]
2025-11-11T17:19:17.176799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518637485952680:2048] [node 1] ICS09 handshake done sender: [2:7571518637485952672:2118] self: [2:7571518637485952675:0] peer: [1:7571518635626231605:0] socket: 307 qp: -1
2025-11-11T17:19:17.176821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518637485952680:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:17.176844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:17.178498Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518631331264298:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:17.178616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518629790195292:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.178662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:17.178808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518629790195292:2068] [node 4] ICP22 created new session: [3:7571518634085162625:2068]
2025-11-11T17:19:17.178844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518634085162625:2068] [node 4] ICS09 handshake done sender: [3:7571518629790195302:2114] self: [3:7571518629790195301:0] peer: [4:7571518634385006989:0] socket: 303 qp: -1
2025-11-11T17:19:17.178875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518634085162625:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:17.178905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:17.179737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518631331264234:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.179805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:17.179838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518631331264234:2048] [node 2] ICP22 created new session: [1:7571518635626231629:2048]
2025-11-11T17:19:17.179872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518635626231629:2048] [node 2] ICS09 handshake done sender: [1:7571518635626231606:2116] self: [1:7571518635626231605:0] peer: [2:7571518637485952675:0] socket: 271 qp: -1
2025-11-11T17:19:17.179896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518635626231629:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:17.179924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:17.180296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518631331264235:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.180356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:17.180405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518631331264235:2049] [node 3] ICP22 created new session: [1:7571518635626231631:2049]
2025-11-11T17:19:17.180438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518635626231631:2049] [node 3] ICS09 handshake done sender: [1:7571518631331264298:2110] self: [1:7571518631331264297:0] peer: [3:7571518634085162622:0] socket: 246 qp: -1
2025-11-11T17:19:17.180462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518635626231631:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:17.180498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:17.182122Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:17.184107Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518634085162614:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.184527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518637577096903:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52989
2025-11-11T17:19:17.184770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518629790195227:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.184809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:17.184846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518629790195227:2048] [node 1] ICP22 created new session: [3:7571518634085162627:2048]
2025-11-11T17:19:17.184882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518634085162627:2048] [node 1] ICS09 handshake done sender: [3:7571518634085162614:2120] self: [3:7571518634085162622:0] peer: [1:7571518631331264297:0] socket: 293 qp: -1
2025-11-11T17:19:17.184906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518634085162627:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:17.184937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:17.186805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518633282129508:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518637577096892:2121] held: no
2025-11-11T17:19:17.186960Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518637577096892:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:17.188946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518633282129508:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.189008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:17.189045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518633282129508:2049] [node 2] ICP22 created new session: [6:7571518637577096906:2049]
2025-11-11T17:19:17.189080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518637577096906:2049] [node 2] ICS09 handshake done sender: [6:7571518637577096892:2121] self: [6:7571518637577096902:0] peer: [2:7571518633190985357:0] socket: 253 qp: -1
2025-11-11T17:19:17.189105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518637577096906:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:17.189133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:17.198087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518634385006990:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.198207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518630090039599:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:17.198540Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518635626231608:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:17.198571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518635626231608:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:17.199635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518634385006995:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6954
2025-11-11T17:19:17.199892Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518635626231608:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:17.200345Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518634385006990:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:17.200545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518630090039599:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:17.200611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:17.200651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518630090039599:2048] [node 1] ICP22 created new session: [4:7571518634385006996:2048]
2025-11-11T17:19:17.200688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518634385006996:2048] [node 1] ICS09 handshake done sender: [4:7571518634385006990:2119] self: [4:7571518634385006994:0] peer: [1:7571518635626231607:0] socket: 310 qp: -1
2025-11-11T17:19:17.200721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518634385006996:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:17.200760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:17.202502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518631331264300:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:17.202569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:17.202616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518631331264300:2089] [node 4] ICP22 created new session: [1:7571518635626231634:2089]
2025-11-11T17:19:17.202653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518635626231634:2089] [node 4] ICS09 handshake done sender: [1:7571518635626231608:2117] self: [1:7571518635626231607:0] peer: [4:7571518634385006994:0] socket: 278 qp: -1
2025-11-11T17:19:17.202682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518635626231634:2089] [node 4] ICS10 traffic start
2025-11-11T17:19:17.202728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:17.245341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:17.245559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518637485952680:2048] [node 1] ICS01 socket: 307 reason# EndOfStream
2025-11-11T17:19:17.245607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518633190985283:2048] [node 1] ICP30 unregister session Session# [2:7571518637485952680:2048] VirtualId# [2:7571518637485952675:0]
2025-11-11T17:19:17.245638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518637485952680:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.245710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:17.245757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:17.247627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:17.247836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518634085162627:2048] [node 1] ICS01 socket: 293 reason# EndOfStream
2025-11-11T17:19:17.247866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518629790195227:2048] [node 1] ICP30 unregister session Session# [3:7571518634085162627:2048] VirtualId# [3:7571518634085162622:0]
2025-11-11T17:19:17.247895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518634085162627:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.247956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:17.247989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:17.252507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:17.252703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518636301656979:2048] [node 1] ICS01 socket: 291 reason# EndOfStream
2025-11-11T17:19:17.252734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518632006689584:2048] [node 1] ICP30 unregister session Session# [5:7571518636301656979:2048] VirtualId# [5:7571518636301656975:0]
2025-11-11T17:19:17.252756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518636301656979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.252807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:17.252842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:17.252909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:17.253081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518634385006996:2048] [node 1] ICS01 socket: 310 reason# EndOfStream
2025-11-11T17:19:17.253107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518630090039599:2048] [node 1] ICP30 unregister session Session# [4:7571518634385006996:2048] VirtualId# [4:7571518634385006994:0]
2025-11-11T17:19:17.253132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518634385006996:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.253172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:17.253209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:17.253261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:17.253429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518637577096904:2048] [node 1] ICS01 socket: 294 reason# EndOfStream
2025-11-11T17:19:17.253452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518633282129507:2048] [node 1] ICP30 unregister session Session# [6:7571518637577096904:2048] VirtualId# [6:7571518637577096895:0]
2025-11-11T17:19:17.253471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518637577096904:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.253506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:17.253532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:17.329749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:17.330003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518634085162607:2049] [node 2] ICS01 socket: 257 reason# EndOfStream
2025-11-11T17:19:17.330057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518629790195228:2049] [node 2] ICP30 unregister session Session# [3:7571518634085162607:2049] VirtualId# [3:7571518629790195309:0]
2025-11-11T17:19:17.330094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518634085162607:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.330210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:17.330254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:17.332059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:17.332312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518634385006987:2049] [node 2] ICS01 socket: 262 reason# EndOfStream
2025-11-11T17:19:17.332359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518630090039600:2049] [node 2] ICP30 unregister session Session# [4:7571518634385006987:2049] VirtualId# [4:7571518634385006976:0]
2025-11-11T17:19:17.332390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518634385006987:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.332468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:17.332510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:17.332598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:17.332792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518636301656963:2049] [node 2] ICS01 socket: 276 reason# EndOfStream
2025-11-11T17:19:17.332819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518632006689585:2049] [node 2] ICP30 unregister session Session# [5:7571518636301656963:2049] VirtualId# [5:7571518636301656961:0]
2025-11-11T17:19:17.332840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518636301656963:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.332905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:17.332961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:17.334091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:17.334274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518637577096906:2049] [node 2] ICS01 socket: 253 reason# EndOfStream
2025-11-11T17:19:17.334313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518633282129508:2049] [node 2] ICP30 unregister session Session# [6:7571518637577096906:2049] VirtualId# [6:7571518637577096902:0]
2025-11-11T17:19:17.334335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518637577096906:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.334397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:17.334434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:17.381084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:17.381327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518637577096890:2050] [node 3] ICS01 socket: 266 reason# EndOfStream
2025-11-11T17:19:17.381366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518633282129509:2050] [node 3] ICP30 unregister session Session# [6:7571518637577096890:2050] VirtualId# [6:7571518637577096884:0]
2025-11-11T17:19:17.381396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518637577096890:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.381468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:17.381502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:17.387457Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:17.387661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518634385006992:2050] [node 3] ICS01 socket: 304 reason# EndOfStream
2025-11-11T17:19:17.387717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518630090039601:2050] [node 3] ICP30 unregister session Session# [4:7571518634385006992:2050] VirtualId# [4:7571518634385006989:0]
2025-11-11T17:19:17.387748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518634385006992:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.387841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:17.387883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:17.385690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:17.385921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518636301656981:2050] [node 3] ICS01 socket: 263 reason# EndOfStream
2025-11-11T17:19:17.396045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518632006689586:2050] [node 3] ICP30 unregister session Session# [5:7571518636301656981:2050] VirtualId# [5:7571518636301656966:0]
2025-11-11T17:19:17.396101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518636301656981:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.396218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:17.396273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:17.523915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:17.524158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518637577096899:2068] [node 4] ICS01 socket: 282 reason# EndOfStream
2025-11-11T17:19:17.524202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518633282129567:2068] [node 4] ICP30 unregister session Session# [6:7571518637577096899:2068] VirtualId# [6:7571518637577096886:0]
2025-11-11T17:19:17.524241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518637577096899:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.524337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:17.524380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:17.524466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:17.524655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518636301656969:2060] [node 4] ICS01 socket: 277 reason# EndOfStream
2025-11-11T17:19:17.524689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518632006689603:2060] [node 4] ICP30 unregister session Session# [5:7571518636301656969:2060] VirtualId# [5:7571518636301656967:0]
2025-11-11T17:19:17.524710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518636301656969:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.524750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:17.524787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:17.548927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:17.549197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518637577096896:2064] [node 5] ICS01 socket: 283 reason# EndOfStream
2025-11-11T17:19:17.549237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518633282129525:2064] [node 5] ICP30 unregister session Session# [6:7571518637577096896:2064] VirtualId# [6:7571518637577096887:0]
2025-11-11T17:19:17.549267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518637577096896:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:17.549357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:17.549397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:17.728870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518635627028559:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:17.728881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518635627028560:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:17.747602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518634718454046:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:17.747835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518634718454045:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:17.821883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518638175798879:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:17.821923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518638175798880:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:17.896507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518635935655573:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:17.896544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518635935655574:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:17.896569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518635935655575:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:17.876097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518637145699210:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:17.876136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518637145699211:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:17.876159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518637145699212:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:17.967155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518634652989332:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:17.967190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518634652989333:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:17.967215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518634652989334:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:18.186129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518641440666533:2072] [node 5] ICP01 ready to work
2025-11-11T17:19:18.189671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518639013421362:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:18.198310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518642470766206:2074] [node 4] ICP01 ready to work
2025-11-11T17:19:18.250449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518639921995894:2083] [node 6] ICP01 ready to work
2025-11-11T17:19:18.221770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518639013421368:2067] [node 5] ICP01 ready to work
2025-11-11T17:19:18.237732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518641440666538:2076] [node 6] ICP01 ready to work
2025-11-11T17:19:18.250879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518639921995902:2084] [node 5] ICP01 ready to work
2025-11-11T17:19:18.339265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518638175798880:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(140656038ns) {Configure:104(140651853ns) {ProcessConfigured:135(55657ns) {StartInitialHandshake:157(52721ns) {DropHandshakes:469(2349ns) {DropIncomingHandshake:441(605ns) DropOutgoingHandshake:456(391ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:476(1326ns) {SwitchToState:178(352ns)}}}}}
2025-11-11T17:19:18.339597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518642470766235:2075] [node 6] ICP01 ready to work
2025-11-11T17:19:18.339889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518642470766236:2077] [node 5] ICP01 ready to work
2025-11-11T17:19:18.297653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518640230622915:2086] [node 4] ICP01 ready to work
2025-11-11T17:19:18.357638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518640230622930:2088] [node 6] ICP01 ready to work
2025-11-11T17:19:18.383576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518641440666533:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(85677237ns) {Configure:104(85672153ns) {ProcessConfigured:135(82578ns) {StartInitialHandshake:157(79618ns) {DropHandshakes:469(2250ns) {DropIncomingHandshake:441(512ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(738ns) PrepareNewSessionHandshake:476(1197ns) {SwitchToState:178(479ns)}}}}}
2025-11-11T17:19:18.251149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518639921995903:2085] [node 4] ICP01 ready to work
2025-11-11T17:19:18.346039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518635627028560:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(91651009ns) {Configure:104(91647976ns) {ProcessConfigured:135(28676718ns) {StartInitialHandshake:157(28668958ns) {DropHandshakes:469(3374ns) {DropIncomingHandshake:441(945ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(928ns) PrepareNewSessionHandshake:476(3797ns) {SwitchToState:178(1853ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.450638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518641440666538:2076] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(65400630ns) {Configure:104(65397436ns) {ProcessConfigured:135(74787ns) {StartInitialHandshake:157(72325ns) {DropHandshakes:469(2110ns) {DropIncomingHandshake:441(600ns) DropOutgoingHandshake:456(349ns)} GenerateSessionVirtualId:988(836ns) PrepareNewSessionHandshake:476(1400ns) {SwitchToState:178(511ns)}}}}}
2025-11-11T17:19:18.452905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518641440666580:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37991
2025-11-11T17:19:18.482967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518634718454045:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(342040410ns) {Configure:104(342035906ns) {ProcessConfigured:135(64027ns) {StartInitialHandshake:157(60836ns) {DropHandshakes:469(2325ns) {DropIncomingHandshake:441(480ns) DropOutgoingHandshake:456(481ns)} GenerateSessionVirtualId:988(903ns) PrepareNewSessionHandshake:476(1347ns) {SwitchToState:178(529ns)}}}}}
2025-11-11T17:19:18.484810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518639013421410:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.526187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518634652989332:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(352214119ns) {Configure:104(352202715ns) {ProcessConfigured:135(66536ns) {StartInitialHandshake:157(60854ns) {DropHandshakes:469(2270ns) {DropIncomingHandshake:441(525ns) DropOutgoingHandshake:456(496ns)} GenerateSessionVirtualId:988(770ns) PrepareNewSessionHandshake:476(1458ns) {SwitchToState:178(587ns)}}}}}
2025-11-11T17:19:18.545681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518642470766252:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43281
2025-11-11T17:19:18.547093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518635627028559:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(417038795ns) {Configure:104(417033684ns) {ProcessConfigured:135(9914043ns) {StartInitialHandshake:157(9910634ns) {DropHandshakes:469(2482ns) {DropIncomingHandshake:441(566ns) DropOutgoingHandshake:456(482ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:476(6517ns) {SwitchToState:178(2034ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.534397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518634652989333:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(333212145ns) {Configure:104(333207636ns) {ProcessConfigured:135(52982ns) {StartInitialHandshake:157(51339ns) {DropHandshakes:469(2714ns) {DropIncomingHandshake:441(438ns) DropOutgoingHandshake:456(607ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:476(1174ns) {SwitchToState:178(259ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.562589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518639013421362:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(288598517ns) {Configure:104(264635424ns) {ProcessConfigured:135(53927ns) {StartInitialHandshake:157(51575ns) {DropHandshakes:469(2625ns) {DropIncomingHandshake:441(974ns) DropOutgoingHandshake:456(455ns)} GenerateSessionVirtualId:988(708ns) PrepareNewSessionHandshake:476(1285ns) {SwitchToState:178(428ns)}}}}}
2025-11-11T17:19:18.538250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518638947956697:2076] [node 5] ICP01 ready to work
2025-11-11T17:19:18.602122Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518642470766252:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.602343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518638175798879:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.603840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518642470766257:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43281
2025-11-11T17:19:18.604497Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518642470766252:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:18.554702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956698:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.554860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956699:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.554967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956700:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.595433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518638947956706:2075] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.598465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518639921995934:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38953
2025-11-11T17:19:18.598595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518639921995935:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38953
2025-11-11T17:19:18.602688Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995925:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.602709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518639921995925:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.604034Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518639921995925:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:18.606308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518635627028560:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.609559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:18.616765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518635627028560:2049] [node 3] ICP22 created new session: [1:7571518639921995938:2049]
2025-11-11T17:19:18.621660Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518638947956699:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.613997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518641440666580:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.629703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518638947956698:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.564319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518639013421411:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:19:18.632401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518635935655574:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(242305302ns) {Configure:104(242301493ns) {ProcessConfigured:135(47694ns) {StartInitialHandshake:157(45278ns) {DropHandshakes:469(1929ns) {DropIncomingHandshake:441(431ns) DropOutgoingHandshake:456(337ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:476(1377ns) {SwitchToState:178(416ns)}}}}}
2025-11-11T17:19:18.615290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518638175798879:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.653784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622942:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.653974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622943:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.654059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622944:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.654138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622945:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.614367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518637145699212:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.633904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518639921995938:2049] [node 3] ICS09 handshake done sender: [1:7571518639921995925:2112] self: [1:7571518639921995923:0] peer: [3:7571518642470766256:0] socket: 255 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.665942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518640230622944:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.667303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518635935655574:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.564527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518639013421412:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.614445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518641440666584:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37991
2025-11-11T17:19:18.634257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518639921995938:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:18.615352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.649466Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518638947956700:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.703033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518638947956706:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(68486903ns) {Configure:104(68482012ns) {ProcessConfigured:135(45716ns) {StartInitialHandshake:157(44082ns) {DropHandshakes:469(4776ns) {DropIncomingHandshake:441(616ns) DropOutgoingHandshake:456(334ns)} GenerateSessionVirtualId:988(675ns) PrepareNewSessionHandshake:476(1875ns) {SwitchToState:178(607ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.703202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518634652989334:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.703278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518634652989332:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.704838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518638947956706:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.704992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.565064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518639013421417:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:19:18.614839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421411:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.614916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421417:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.614982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421412:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.662199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518639013421424:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:19:18.690068Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421409:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.690092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518639013421409:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.705548Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421424:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.710287Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518638947956710:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.710451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518638947956697:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.648986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518637145699210:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(466837550ns) {Configure:104(466832111ns) {ProcessConfigured:135(14904924ns) {StartInitialHandshake:157(14901574ns) {DropHandshakes:469(368535ns) {DropIncomingHandshake:441(346ns) DropOutgoingHandshake:456(1079ns)} GenerateSessionVirtualId:988(878ns) PrepareNewSessionHandshake:476(3446ns) {SwitchToState:178(1863ns)}}}}}
2025-11-11T17:19:18.615396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518638175798879:2048] [node 1] ICP22 created new session: [3:7571518642470766258:2048]
2025-11-11T17:19:18.645070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.693812Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995935:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.615437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518642470766258:2048] [node 1] ICS09 handshake done sender: [3:7571518642470766252:2114] self: [3:7571518642470766256:0] peer: [1:7571518639921995923:0] socket: 262 qp: -1
2025-11-11T17:19:18.717771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956707:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.721858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518634718454046:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(199507917ns) {Configure:104(199503639ns) {ProcessConfigured:135(42825828ns) {StartInitialHandshake:157(42822596ns) {DropHandshakes:469(2408ns) {DropIncomingHandshake:441(514ns) DropOutgoingHandshake:456(299ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:476(3056ns) {SwitchToState:178(1277ns)}}}}}
2025-11-11T17:19:18.703615Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518640230622943:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.754539Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4521) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.694515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518641440666588:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37991
2025-11-11T17:19:18.703728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518640230622942:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.615469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518642470766258:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:18.729667Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518639013421409:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:18.697983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518635627028560:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(93009091ns) {HandleHandshakeStatus:329(93007142ns) {DropIncomingHandshake:441(1187ns) DropOutgoingHandshake:456(655ns) SwitchToState:178(356ns) ProcessPendingSessionEvents:495(23942886ns) {ForwardSessionEventToSession:612(23885834ns) ForwardSessionEventToSession:612(7028ns) ForwardSessionEventToSession:612(7083ns) ForwardSessionEventToSession:612(3349ns) ForwardSessionEventToSession:612(2972ns) ForwardSessionEventToSession:612(5287ns) ForwardSessionEventToSession:612(3152ns) ForwardSessionEventToSession:612(1921ns) ForwardSessionEventToSession:612(9363ns) ForwardSessionEventToSession:612(2481ns)}}}
2025-11-11T17:19:18.730820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518639013421368:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.730853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:18.730886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518639013421368:2067] [node 5] ICP22 created new session: [2:7571518639013421429:2067]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.786147Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518638947956707:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.786229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518634652989334:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4521) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518638947956699:2114] held: no
2025-11-11T17:19:18.786817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956715:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.722840Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518641440666577:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.730921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518639013421429:2067] [node 5] ICS09 handshake done sender: [2:7571518639013421409:2111] self: [2:7571518639013421408:0] peer: [5:7571518640230622950:0] socket: 274 qp: -1
2025-11-11T17:19:18.615501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.703841Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518640230622945:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.701752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995934:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.731152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518639013421429:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:18.722860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518641440666577:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.723569Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518641440666580:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:18.723863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518637145699212:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.723898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:18.723933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518637145699212:2050] [node 3] ICP22 created new session: [4:7571518641440666590:2050]
2025-11-11T17:19:18.723991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518641440666590:2050] [node 3] ICS09 handshake done sender: [4:7571518641440666580:2115] self: [4:7571518641440666585:0] peer: [3:7571518642470766237:0] socket: 258 qp: -1
2025-11-11T17:19:18.724020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518641440666590:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:18.724060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:18.725192Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518641440666577:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:18.725907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518641440666584:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.725958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518641440666538:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.733828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:18.739068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518637145699210:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.749695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518641440666538:2076] [node 6] ICP22 created new session: [4:7571518641440666593:2076]
2025-11-11T17:19:18.762204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518641440666593:2076] [node 6] ICS09 handshake done sender: [4:7571518641440666577:2113] self: [4:7571518641440666576:0] peer: [6:7571518638947956713:0] socket: 261 qp: -1
2025-11-11T17:19:18.769716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518641440666593:2076] [node 6] ICS10 traffic start
2025-11-11T17:19:18.773746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:18.793943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518641440666538:2076] [node 6] ICP35 event processing took too much time PendingConnection:278(67790953ns) {HandleHandshakeStatus:329(67789157ns) {DropIncomingHandshake:441(196ns) DropOutgoingHandshake:456(335ns) SwitchToState:178(285ns) ProcessPendingSessionEvents:495(8005516ns) {ForwardSessionEventToSession:612(7941810ns) ForwardSessionEventToSession:612(12511ns) ForwardSessionEventToSession:612(3361ns) ForwardSessionEventToSession:612(5606ns) ForwardSessionEventToSession:612(2142ns) ForwardSessionEventToSession:612(9789ns) ForwardSessionEventToSession:612(2562ns) ForwardSessionEventToSession:612(2785ns) ForwardSessionEventToSession:612(4504ns) ForwardSessionEventToSession:612(5328ns)}}}
2025-11-11T17:19:18.794420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518641440666595:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37991
2025-11-11T17:19:18.794600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518641440666597:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37991
2025-11-11T17:19:18.805897Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518641440666584:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:18.806134Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.807133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518637145699210:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.807180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:18.807224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518637145699210:2048] [node 1] ICP22 created new session: [4:7571518641440666598:2048]
2025-11-11T17:19:18.807265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518641440666598:2048] [node 1] ICS09 handshake done sender: [4:7571518641440666584:2116] self: [4:7571518641440666594:0] peer: [1:7571518639921995926:0] socket: 282 qp: -1
2025-11-11T17:19:18.807298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518641440666598:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:18.807336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:18.807543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518637145699210:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:18.809739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518641440666595:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.820429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518637145699211:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.630015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518642470766238:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.630041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518642470766238:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.634109Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518642470766238:2106] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:18.711097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518642470766206:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.711132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:18.711173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518642470766206:2074] [node 4] ICP22 created new session: [3:7571518642470766263:2074]
2025-11-11T17:19:18.711214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518642470766263:2074] [node 4] ICS09 handshake done sender: [3:7571518642470766238:2106] self: [3:7571518642470766237:0] peer: [4:7571518641440666585:0] socket: 256 qp: -1
2025-11-11T17:19:18.711238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518642470766263:2074] [node 4] ICS10 traffic start
2025-11-11T17:19:18.711272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:18.711929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518642470766240:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.711946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518642470766240:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.712721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518642470766255:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43281
2025-11-11T17:19:18.713016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518642470766262:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43281
2025-11-11T17:19:18.717974Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518642470766240:2107] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:18.722016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518642470766235:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.722043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:18.722081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518642470766235:2075] [node 6] ICP22 created new session: [3:7571518642470766266:2075]
2025-11-11T17:19:18.722119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518642470766266:2075] [node 6] ICS09 handshake done sender: [3:7571518642470766240:2107] self: [3:7571518642470766239:0] peer: [6:7571518638947956711:0] socket: 257 qp: -1
2025-11-11T17:19:18.722148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518642470766266:2075] [node 6] ICS10 traffic start
2025-11-11T17:19:18.722171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:18.722481Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518642470766262:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.730087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518642470766255:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.730177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518642470766235:2075] [node 6] ICP18 handshake (actor [3:7571518642470766255:2116]) is too old
2025-11-11T17:19:18.749732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518642470766242:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.749769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518642470766242:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.752956Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518642470766242:2108] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:18.753196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518642470766236:2077] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.753285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:18.753331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518642470766236:2077] [node 5] ICP22 created new session: [3:7571518642470766270:2077]
2025-11-11T17:19:18.753376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518642470766270:2077] [node 5] ICS09 handshake done sender: [3:7571518642470766242:2108] self: [3:7571518642470766241:0] peer: [5:7571518640230622954:0] socket: 283 qp: -1
2025-11-11T17:19:18.753426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518642470766270:2077] [node 5] ICS10 traffic start
2025-11-11T17:19:18.753481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.704119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622952:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.704185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518635935655573:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.704231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518635935655575:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.713437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518640230622939:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.713455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518640230622939:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.725260Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518640230622939:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:18.754679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518640230622915:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(312390247ns) {Configure:104(312385119ns) {ProcessConfigured:135(44817649ns) {StartInitialHandshake:157(44815403ns) {DropHandshakes:469(1890ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(373ns)} GenerateSessionVirtualId:988(868ns) PrepareNewSessionHandshake:476(2799ns) {SwitchToState:178(352ns)}}}}}
2025-11-11T17:19:18.777881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518640230622930:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.777951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:18.778000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518640230622930:2088] [node 6] ICP22 created new session: [5:7571518640230622957:2088]
2025-11-11T17:19:18.778039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518640230622957:2088] [node 6] ICS09 handshake done sender: [5:7571518640230622939:2112] self: [5:7571518640230622938:0] peer: [6:7571518638947956714:0] socket: 292 qp: -1
2025-11-11T17:19:18.778071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518640230622957:2088] [node 6] ICS10 traffic start
2025-11-11T17:19:18.778113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:18.778524Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.831246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518635935655574:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518640230622944:2116] held: no
2025-11-11T17:19:18.831505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518640230622915:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.831733Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4521) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.718019Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995907:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.731191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:18.813896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956716:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.849947Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518640230622944:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:18.718042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518639921995907:2095] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.719625Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518639921995907:2095] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:18.719871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518639921995902:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.719909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:18.719969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518639921995902:2084] [node 5] ICP22 created new session: [1:7571518639921995943:2084]
2025-11-11T17:19:18.720017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518639921995943:2084] [node 5] ICS09 handshake done sender: [1:7571518639921995907:2095] self: [1:7571518639921995906:0] peer: [5:7571518640230622953:0] socket: 254 qp: -1
2025-11-11T17:19:18.720057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518639921995943:2084] [node 5] ICS10 traffic start
2025-11-11T17:19:18.720124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:18.737679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995905:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.737712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518639921995905:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.751579Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995927:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.751599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518639921995927:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.752448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518639921995946:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38953
2025-11-11T17:19:18.761882Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518639921995927:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:18.790600Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518639921995905:2094] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:18.799256Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995933:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.799278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518639921995933:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.800102Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518639921995946:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.800151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518639921995903:2085] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.800186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:18.800227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518639921995903:2085] [node 4] ICP22 created new session: [1:7571518639921995949:2085]
2025-11-11T17:19:18.800270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518639921995949:2085] [node 4] ICS09 handshake done sender: [1:7571518639921995927:2113] self: [1:7571518639921995926:0] peer: [4:7571518641440666594:0] socket: 281 qp: -1
2025-11-11T17:19:18.800303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518639921995949:2085] [node 4] ICS10 traffic start
2025-11-11T17:19:18.800338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:18.800964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518639921995903:2085] [node 4] ICP18 handshake (actor [1:7571518639921995946:2117]) is too old
2025-11-11T17:19:18.801628Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518639921995933:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:18.818227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518639921995894:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.818280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:18.818320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518639921995894:2083] [node 6] ICP22 created new session: [1:7571518639921995952:2083]
2025-11-11T17:19:18.818387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518639921995952:2083] [node 6] ICS09 handshake done sender: [1:7571518639921995905:2094] self: [1:7571518639921995904:0] peer: [6:7571518638947956712:0] socket: 253 qp: -1
2025-11-11T17:19:18.818423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518639921995952:2083] [node 6] ICS10 traffic start
2025-11-11T17:19:18.818453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:18.819567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518635627028559:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.819627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:18.819667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518635627028559:2048] [node 2] ICP22 created new session: [1:7571518639921995954:2048]
2025-11-11T17:19:18.819722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518639921995954:2048] [node 2] ICS09 handshake done sender: [1:7571518639921995933:2114] self: [1:7571518639921995932:0] peer: [2:7571518639013421432:0] socket: 301 qp: -1
2025-11-11T17:19:18.819750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518639921995954:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:18.819781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.731757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518639013421427:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:19:18.831007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956717:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.832861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956718:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.860274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518635935655575:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4521) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518640230622945:2117] held: no
2025-11-11T17:19:18.745537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518639013421368:2067] [node 5] ICP18 handshake (actor [2:7571518639013421424:2117]) is too old
2025-11-11T17:19:18.833271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518634652989333:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.833325Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518638947956699:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:18.833535Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518638947956710:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:18.837771Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518638947956700:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:18.846176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518638947956698:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:18.857834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518634652989334:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.857915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:18.857961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518634652989334:2050] [node 3] ICP22 created new session: [6:7571518638947956720:2050]
2025-11-11T17:19:18.858007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518638947956720:2050] [node 3] ICS09 handshake done sender: [6:7571518638947956699:2114] self: [6:7571518638947956711:0] peer: [3:7571518642470766239:0] socket: 266 qp: -1
2025-11-11T17:19:18.858037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518638947956720:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:18.858068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:18.858396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518638947956697:2076] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.858423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:18.858458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518638947956697:2076] [node 5] ICP22 created new session: [6:7571518638947956723:2076]
2025-11-11T17:19:18.858492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518638947956723:2076] [node 5] ICS09 handshake done sender: [6:7571518638947956710:2119] self: [6:7571518638947956714:0] peer: [5:7571518640230622938:0] socket: 296 qp: -1
2025-11-11T17:19:18.858522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518638947956723:2076] [node 5] ICS10 traffic start
2025-11-11T17:19:18.858548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:18.858928Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.875153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518638947956706:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.875204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:18.875265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518638947956706:2075] [node 4] ICP22 created new session: [6:7571518638947956725:2075]
2025-11-11T17:19:18.875308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518638947956725:2075] [node 4] ICS09 handshake done sender: [6:7571518638947956700:2115] self: [6:7571518638947956713:0] peer: [4:7571518641440666576:0] socket: 267 qp: -1
2025-11-11T17:19:18.875342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518638947956725:2075] [node 4] ICS10 traffic start
2025-11-11T17:19:18.875373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:18.876463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518634652989332:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518638947956698:2113] held: no
2025-11-11T17:19:18.876546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518634652989332:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.876577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:18.876620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518634652989332:2048] [node 1] ICP22 created new session: [6:7571518638947956727:2048]
2025-11-11T17:19:18.889789Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518641440666571:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.876662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518638947956727:2048] [node 1] ICS09 handshake done sender: [6:7571518638947956698:2113] self: [6:7571518638947956712:0] peer: [1:7571518639921995904:0] socket: 265 qp: -1
2025-11-11T17:19:18.876688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518638947956727:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:18.876717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:18.891379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518638947956729:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61518
2025-11-11T17:19:18.891047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518642470766272:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43281
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.770017Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421427:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.860438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518635935655574:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.770182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518634718454045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.825792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518639013421433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:19:18.852677Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.853916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421416:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.853938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518639013421416:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.854600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518634718454045:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518639013421427:2119] held: no
2025-11-11T17:19:18.854763Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518639013421427:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:18.861420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518634718454045:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.861478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:18.861513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518634718454045:2048] [node 1] ICP22 created new session: [2:7571518639013421435:2048]
2025-11-11T17:19:18.861544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518639013421435:2048] [node 1] ICS09 handshake done sender: [2:7571518639013421427:2119] self: [2:7571518639013421432:0] peer: [1:7571518639921995932:0] socket: 307 qp: -1
2025-11-11T17:19:18.861565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518639013421435:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:18.861606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:18.866096Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421414:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.866116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518639013421414:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.886379Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518639013421416:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:18.905954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518639013421410:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.906008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:18.906059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518639013421410:2109] [node 4] ICP22 created new session: [2:7571518639013421439:2109]
2025-11-11T17:19:18.906102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518639013421439:2109] [node 4] ICS09 handshake done sender: [2:7571518639013421416:2115] self: [2:7571518639013421415:0] peer: [4:7571518641440666600:0] socket: 271 qp: -1
2025-11-11T17:19:18.906132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518639013421439:2109] [node 4] ICS10 traffic start
2025-11-11T17:19:18.906163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:18.917979Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518639013421414:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:18.889828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518641440666571:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.860485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:18.930336Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518638947956707:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:18.944864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518634652989333:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.897818Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518641440666571:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:18.943298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518639013421362:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.953838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518639921995956:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38953
2025-11-11T17:19:18.953910Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:18.917716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518641440666533:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.917795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:18.917843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518641440666533:2072] [node 5] ICP22 created new session: [4:7571518641440666602:2072]
2025-11-11T17:19:18.917902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518641440666602:2072] [node 5] ICS09 handshake done sender: [4:7571518641440666571:2110] self: [4:7571518641440666570:0] peer: [5:7571518640230622960:0] socket: 259 qp: -1
2025-11-11T17:19:18.930533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518641440666602:2072] [node 5] ICS10 traffic start
2025-11-11T17:19:18.937834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:18.938257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518641440666533:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(28364877ns) {HandleHandshakeStatus:329(28362613ns) {DropIncomingHandshake:441(1187ns) DropOutgoingHandshake:456(966ns) SwitchToState:178(627ns) ProcessPendingSessionEvents:495(93503ns) {ForwardSessionEventToSession:612(49727ns) ForwardSessionEventToSession:612(3361ns) ForwardSessionEventToSession:612(5020ns) ForwardSessionEventToSession:612(2177ns) ForwardSessionEventToSession:612(5206ns) ForwardSessionEventToSession:612(4575ns) ForwardSessionEventToSession:612(2781ns) ForwardSessionEventToSession:612(4584ns) ForwardSessionEventToSession:612(3436ns) ForwardSessionEventToSession:612(2764ns)}}}
2025-11-11T17:19:18.938387Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.958330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518637145699211:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518641440666595:2119] held: no
2025-11-11T17:19:18.958491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518641440666604:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37991
2025-11-11T17:19:18.941683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518642470766272:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.977790Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518641440666604:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.978201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518641440666533:2072] [node 5] ICP18 handshake (actor [4:7571518641440666604:2122]) is too old
2025-11-11T17:19:18.943368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:18.943419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518639013421362:2065] [node 6] ICP22 created new session: [2:7571518639013421441:2065]
2025-11-11T17:19:18.943464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518639013421441:2065] [node 6] ICS09 handshake done sender: [2:7571518639013421414:2114] self: [2:7571518639013421413:0] peer: [6:7571518638947956719:0] socket: 275 qp: -1
2025-11-11T17:19:18.943514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518639013421441:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:18.943550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:18.966482Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518639013421425:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:18.966506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518639013421425:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:18.970808Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518639013421425:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:18.860524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518635935655574:2049] [node 2] ICP22 created new session: [5:7571518640230622962:2049]
2025-11-11T17:19:18.860560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518640230622962:2049] [node 2] ICS09 handshake done sender: [5:7571518640230622944:2116] self: [5:7571518640230622950:0] peer: [2:7571518639013421408:0] socket: 287 qp: -1
2025-11-11T17:19:18.860584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518640230622962:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:18.860608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:18.861077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622956:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.861972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622959:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.862545Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518640230622942:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:18.862725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518640230622945:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:18.862898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518635935655573:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.862934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:18.862980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518635935655573:2048] [node 1] ICP22 created new session: [5:7571518640230622965:2048]
2025-11-11T17:19:18.863014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518640230622965:2048] [node 1] ICS09 handshake done sender: [5:7571518640230622942:2114] self: [5:7571518640230622953:0] peer: [1:7571518639921995906:0] socket: 285 qp: -1
2025-11-11T17:19:18.863035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518640230622965:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:18.863059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:18.863432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518635935655575:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.863459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:18.863489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518635935655575:2050] [node 3] ICP22 created new session: [5:7571518640230622967:2050]
2025-11-11T17:19:18.863518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518640230622967:2050] [node 3] ICS09 handshake done sender: [5:7571518640230622945:2117] self: [5:7571518640230622954:0] peer: [3:7571518642470766241:0] socket: 288 qp: -1
2025-11-11T17:19:18.863538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518640230622967:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:18.863564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:18.922589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622969:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.922643Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:18.925896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518640230622970:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3630
2025-11-11T17:19:18.958719Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518640230622943:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:18.970133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518640230622915:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:18.970178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:18.970223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518640230622915:2086] [node 4] ICP22 created new session: [5:7571518640230622971:2086]
2025-11-11T17:19:18.970266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518640230622971:2086] [node 4] ICS09 handshake done sender: [5:7571518640230622943:2115] self: [5:7571518640230622960:0] peer: [4:7571518641440666570:0] socket: 286 qp: -1
2025-11-11T17:19:18.970296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518640230622971:2086] [node 4] ICS10 traffic start
2025-11-11T17:19:18.970331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:18.980369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518641440666603:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37991
2025-11-11T17:19:18.944907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:18.961661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518634652989333:2049] [node 2] ICP22 created new session: [6:7571518638947956730:2049]
2025-11-11T17:19:18.945832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518638175798880:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:18.961743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518638947956730:2049] [node 2] ICS09 handshake done sender: [6:7571518638947956707:2117] self: [6:7571518638947956719:0] peer: [2:7571518639013421413:0] socket: 276 qp: -1
2025-11-11T17:19:18.961784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518638947956730:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:18.981784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518634718454046:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:18.981844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:18.981887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518634718454046:2049] [node 3] ICP22 created new session: [2:7571518639013421444:2049]
2025-11-11T17:19:18.981923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518639013421444:2049] [node 3] ICS09 handshake done sender: [2:7571518639013421425:2118] self: [2:7571518639013421423:0] peer: [3:7571518642470766273:0] socket: 273 qp: -1
2025-11-11T17:19:18.981964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518639013421444:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:18.981997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:18.978662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518642470766274:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43281
2025-11-11T17:19:18.982738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518642470766272:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:19.001844Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:18.961828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:18.962222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518634652989333:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17200666ns) {HandleHandshakeStatus:329(17198954ns) {DropIncomingHandshake:441(390ns) DropOutgoingHandshake:456(4702ns) SwitchToState:178(468ns) ProcessPendingSessionEvents:495(96981ns) {ForwardSessionEventToSession:612(56279ns) ForwardSessionEventToSession:612(5312ns) ForwardSessionEventToSession:612(5702ns) ForwardSessionEventToSession:612(2855ns) ForwardSessionEventToSession:612(5272ns) ForwardSessionEventToSession:612(2427ns) ForwardSessionEventToSession:612(2514ns) ForwardSessionEventToSession:612(4548ns) ForwardSessionEventToSession:612(1576ns) ForwardSessionEventToSession:612(3928ns)}}}
2025-11-11T17:19:19.009997Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518641440666595:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:19.025803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518638175798880:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:19.025877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:19.025931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518638175798880:2049] [node 2] ICP22 created new session: [3:7571518646765733571:2049]
2025-11-11T17:19:19.025980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518646765733571:2049] [node 2] ICS09 handshake done sender: [3:7571518642470766272:2119] self: [3:7571518642470766273:0] peer: [2:7571518639013421423:0] socket: 268 qp: -1
2025-11-11T17:19:19.026019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518646765733571:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:19.026085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:19.026359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518638175798880:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:19.022076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518637145699211:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:19.026809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:19.026860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518637145699211:2049] [node 2] ICP22 created new session: [4:7571518645735633902:2049]
2025-11-11T17:19:19.026915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518645735633902:2049] [node 2] ICS09 handshake done sender: [4:7571518641440666595:2119] self: [4:7571518641440666600:0] peer: [2:7571518639013421415:0] socket: 308 qp: -1
2025-11-11T17:19:19.026948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518645735633902:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:19.026977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:19.131015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:19.131233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518639013421435:2048] [node 1] ICS01 socket: 307 reason# EndOfStream
2025-11-11T17:19:19.131270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518634718454045:2048] [node 1] ICP30 unregister session Session# [2:7571518639013421435:2048] VirtualId# [2:7571518639013421432:0]
2025-11-11T17:19:19.131301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518639013421435:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.131373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:19.131407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:19.134023Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:19.134237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518642470766258:2048] [node 1] ICS01 socket: 262 reason# EndOfStream
2025-11-11T17:19:19.134273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518638175798879:2048] [node 1] ICP30 unregister session Session# [3:7571518642470766258:2048] VirtualId# [3:7571518642470766256:0]
2025-11-11T17:19:19.134313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518642470766258:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.134373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:19.134407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:19.142616Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:19.142824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518641440666598:2048] [node 1] ICS01 socket: 282 reason# EndOfStream
2025-11-11T17:19:19.142865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518637145699210:2048] [node 1] ICP30 unregister session Session# [4:7571518641440666598:2048] VirtualId# [4:7571518641440666594:0]
2025-11-11T17:19:19.142889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518641440666598:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.142936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:19.142966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:19.145698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:19.145887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518640230622965:2048] [node 1] ICS01 socket: 285 reason# EndOfStream
2025-11-11T17:19:19.145915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518635935655573:2048] [node 1] ICP30 unregister session Session# [5:7571518640230622965:2048] VirtualId# [5:7571518640230622953:0]
2025-11-11T17:19:19.145941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518640230622965:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.146005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:19.146037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:19.142111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:19.142318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518638947956727:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-11-11T17:19:19.142354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518634652989332:2048] [node 1] ICP30 unregister session Session# [6:7571518638947956727:2048] VirtualId# [6:7571518638947956712:0]
2025-11-11T17:19:19.142387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518638947956727:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.142451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:19.142491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:19.245800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:19.246066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518646765733571:2049] [node 2] ICS01 socket: 268 reason# EndOfStream
2025-11-11T17:19:19.246107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518638175798880:2049] [node 2] ICP30 unregister session Session# [3:7571518646765733571:2049] VirtualId# [3:7571518642470766273:0]
2025-11-11T17:19:19.246139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518646765733571:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.246234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:19.246273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:19.247888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:19.248078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518638947956730:2049] [node 2] ICS01 socket: 276 reason# EndOfStream
2025-11-11T17:19:19.248104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518634652989333:2049] [node 2] ICP30 unregister session Session# [6:7571518638947956730:2049] VirtualId# [6:7571518638947956719:0]
2025-11-11T17:19:19.248128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518638947956730:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.248187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:19.248215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:19.249711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:19.249886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518640230622962:2049] [node 2] ICS01 socket: 287 reason# EndOfStream
2025-11-11T17:19:19.249912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518635935655574:2049] [node 2] ICP30 unregister session Session# [5:7571518640230622962:2049] VirtualId# [5:7571518640230622950:0]
2025-11-11T17:19:19.249934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518640230622962:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.249984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:19.250011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:19.253716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:19.253877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518645735633902:2049] [node 2] ICS01 socket: 308 reason# EndOfStream
2025-11-11T17:19:19.253905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518637145699211:2049] [node 2] ICP30 unregister session Session# [4:7571518645735633902:2049] VirtualId# [4:7571518641440666600:0]
2025-11-11T17:19:19.253938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518645735633902:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.253998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:19.254033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:19.325809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:19.326024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518641440666590:2050] [node 3] ICS01 socket: 258 reason# EndOfStream
2025-11-11T17:19:19.326065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518637145699212:2050] [node 3] ICP30 unregister session Session# [4:7571518641440666590:2050] VirtualId# [4:7571518641440666585:0]
2025-11-11T17:19:19.326100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518641440666590:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.326178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:19.326219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:19.326296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:19.326460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518638947956720:2050] [node 3] ICS01 socket: 266 reason# EndOfStream
2025-11-11T17:19:19.326492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518634652989334:2050] [node 3] ICP30 unregister session Session# [6:7571518638947956720:2050] VirtualId# [6:7571518638947956711:0]
2025-11-11T17:19:19.326512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518638947956720:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.326560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:19.326595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:19.326668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:19.326827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518640230622967:2050] [node 3] ICS01 socket: 288 reason# EndOfStream
2025-11-11T17:19:19.326850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518635935655575:2050] [node 3] ICP30 unregister session Session# [5:7571518640230622967:2050] VirtualId# [5:7571518640230622954:0]
2025-11-11T17:19:19.326869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518640230622967:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.326938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:19.326978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:19.429752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:19.429969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518638947956725:2075] [node 4] ICS01 socket: 267 reason# EndOfStream
2025-11-11T17:19:19.430003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518638947956706:2075] [node 4] ICP30 unregister session Session# [6:7571518638947956725:2075] VirtualId# [6:7571518638947956713:0]
2025-11-11T17:19:19.430035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518638947956725:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.430104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:19.430138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:19.433682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:19.433904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518640230622971:2086] [node 4] ICS01 socket: 286 reason# EndOfStream
2025-11-11T17:19:19.433942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518640230622915:2086] [node 4] ICP30 unregister session Session# [5:7571518640230622971:2086] VirtualId# [5:7571518640230622960:0]
2025-11-11T17:19:19.433976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518640230622971:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.434057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:19.434098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:19.473726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:19.473981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518638947956723:2076] [node 5] ICS01 socket: 296 reason# EndOfStream
2025-11-11T17:19:19.474017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518638947956697:2076] [node 5] ICP30 unregister session Session# [6:7571518638947956723:2076] VirtualId# [6:7571518638947956714:0]
2025-11-11T17:19:19.474044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518638947956723:2076] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:19.474124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:19.474161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:19.662592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518645591050205:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:19.662644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518645591050206:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:19.682108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518646527987791:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:19.682147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518646527987792:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:19.710023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518643849750118:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:19.710070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518643849750119:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:19.722855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518642879438524:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:19.722889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518642879438525:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:19.722914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518642879438526:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:19.798306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518646523033759:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:19.798344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518646523033760:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:19.798367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518646523033761:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:19.985468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518642879438526:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19514736ns) {Configure:104(19510246ns) {ProcessConfigured:135(48053ns) {StartInitialHandshake:157(45107ns) {DropHandshakes:469(2654ns) {DropIncomingHandshake:441(527ns) DropOutgoingHandshake:456(403ns)} GenerateSessionVirtualId:988(979ns) PrepareNewSessionHandshake:476(1411ns) {SwitchToState:178(524ns)}}}}}
2025-11-11T17:19:20.050276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518647174405851:2064] [node 6] ICP01 ready to work
2025-11-11T17:19:19.958134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518642777318600:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:19.966829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518642777318601:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:19.966864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518642777318602:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:19.967493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518642777318617:2061] [node 5] ICP01 ready to work
2025-11-11T17:19:19.986519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518642777318600:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18828738ns) {Configure:104(18825489ns) {ProcessConfigured:135(33201ns) {StartInitialHandshake:157(31401ns) {DropHandshakes:469(1047ns) {DropIncomingHandshake:441(291ns) DropOutgoingHandshake:456(341ns)} GenerateSessionVirtualId:988(345ns) PrepareNewSessionHandshake:476(558ns) {SwitchToState:178(119ns)}}}}}
2025-11-11T17:19:20.042411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518650822955112:2059] [node 6] ICP01 ready to work
2025-11-11T17:19:20.072089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518650818001074:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:20.117666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518650822955144:2061] [node 5] ICP01 ready to work
2025-11-11T17:19:20.117739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518650818001077:2062] [node 4] ICP01 ready to work
2025-11-11T17:19:20.142510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518649886017564:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:20.237861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518647174405894:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:20.133872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518650822955150:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.181825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518649886017566:2062] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.159739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518646527987792:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(176807290ns) {Configure:104(176803137ns) {ProcessConfigured:135(53582ns) {StartInitialHandshake:157(51407ns) {DropHandshakes:469(2490ns) {DropIncomingHandshake:441(468ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(1800ns) {SwitchToState:178(586ns)}}}}}
2025-11-11T17:19:20.213836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518648144717479:2060] [node 5] ICP01 ready to work
2025-11-11T17:19:20.186357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518646527987791:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26391271ns) {Configure:104(26387975ns) {ProcessConfigured:135(59283ns) {StartInitialHandshake:157(56996ns) {DropHandshakes:469(5416ns) {DropIncomingHandshake:441(589ns) DropOutgoingHandshake:456(414ns)} GenerateSessionVirtualId:988(688ns) PrepareNewSessionHandshake:476(1307ns) {SwitchToState:178(404ns)}}}}}
2025-11-11T17:19:20.261899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518650822955164:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31237
2025-11-11T17:19:20.319166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518650822955112:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(121734744ns) {Configure:104(121729181ns) {ProcessConfigured:135(62575ns) {StartInitialHandshake:157(60300ns) {DropHandshakes:469(2199ns) {DropIncomingHandshake:441(430ns) DropOutgoingHandshake:456(448ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:476(1524ns) {SwitchToState:178(558ns)}}}}}
2025-11-11T17:19:20.193916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518649886017565:2063] [node 5] ICP01 ready to work
2025-11-11T17:19:20.255918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518648144717480:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:20.289222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518643849750119:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(126017117ns) {Configure:104(126013677ns) {ProcessConfigured:135(87629ns) {StartInitialHandshake:157(84882ns) {DropHandshakes:469(2579ns) {DropIncomingHandshake:441(557ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(829ns) PrepareNewSessionHandshake:476(1623ns) {SwitchToState:178(618ns)}}}}}
2025-11-11T17:19:20.339974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518648144717481:2071] [node 6] ICP01 ready to work
2025-11-11T17:19:20.244321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518649886017564:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(39625698ns) {Configure:104(39621756ns) {ProcessConfigured:135(37639464ns) {StartInitialHandshake:157(37637841ns) {DropHandshakes:469(1984ns) {DropIncomingHandshake:441(452ns) DropOutgoingHandshake:456(299ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:476(2908ns) {SwitchToState:178(1356ns)}}}}}
2025-11-11T17:19:20.314534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518647174405898:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13912
2025-11-11T17:19:20.314770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518647174405897:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13912
2025-11-11T17:19:20.337797Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518647174405897:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.369911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518646523033759:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(267645678ns) {Configure:104(267641128ns) {ProcessConfigured:135(16606188ns) {StartInitialHandshake:157(16602826ns) {DropHandshakes:469(2724ns) {DropIncomingHandshake:441(529ns) DropOutgoingHandshake:456(551ns)} GenerateSessionVirtualId:988(959ns) PrepareNewSessionHandshake:476(4621ns) {SwitchToState:178(2440ns)}}}}}
2025-11-11T17:19:20.259831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518649886017567:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8530
2025-11-11T17:19:20.394104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518642777318601:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(284130901ns) {Configure:104(284126378ns) {ProcessConfigured:135(60171ns) {StartInitialHandshake:157(57062ns) {DropHandshakes:469(2455ns) {DropIncomingHandshake:441(706ns) DropOutgoingHandshake:456(369ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:476(1269ns) {SwitchToState:178(481ns)}}}}}
2025-11-11T17:19:20.259897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518649886017568:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8530
2025-11-11T17:19:20.259951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518649886017569:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8530
2025-11-11T17:19:20.397211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518647072285970:2070] [node 4] ICP01 ready to work
2025-11-11T17:19:20.321642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017568:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.330012Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017569:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.401987Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017573:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.402029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518649886017573:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.403778Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518649886017573:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.397958Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518647174405898:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.398088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518642879438524:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.402915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955164:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.427521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001122:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.425676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518649886017564:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(24402465ns) {IncomingHandshake:261(11645689ns) {DropIncomingHandshake:441(559ns) PrepareNewSessionHandshake:476(1210ns) {SwitchToState:178(673ns)} GenerateSessionVirtualId:988(753ns) IssueIncomingHandshakeReply:193(9875ns)}}
2025-11-11T17:19:20.403059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518646527987791:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.398528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518642879438525:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(321053340ns) {Configure:104(321048710ns) {ProcessConfigured:135(22522614ns) {StartInitialHandshake:157(22519152ns) {DropHandshakes:469(2454ns) {DropIncomingHandshake:441(517ns) DropOutgoingHandshake:456(423ns)} GenerateSessionVirtualId:988(1001ns) PrepareNewSessionHandshake:476(3204ns) {SwitchToState:178(1249ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.474175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518643849750118:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(218007267ns) {Configure:104(218002521ns) {ProcessConfigured:135(82010ns) {StartInitialHandshake:157(79556ns) {DropHandshakes:469(4960ns) {DropIncomingHandshake:441(497ns) DropOutgoingHandshake:456(364ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:476(1121ns) {SwitchToState:178(416ns)}}}}}
2025-11-11T17:19:20.446076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518650822955168:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31237
2025-11-11T17:19:20.427691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001123:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.453806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518649886017564:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.518670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518647072285970:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(121153130ns) {Configure:104(121148131ns) {ProcessConfigured:135(24690506ns) {StartInitialHandshake:157(24688058ns) {DropHandshakes:469(11801341ns) {DropIncomingHandshake:441(464ns) DropOutgoingHandshake:456(2165ns)} GenerateSessionVirtualId:988(1063ns) PrepareNewSessionHandshake:476(1813ns) {SwitchToState:178(809ns)}}}}}
2025-11-11T17:19:20.529661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285969:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.403443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518647174405902:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.476135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518648144717483:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57102
2025-11-11T17:19:20.457627Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955160:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.453874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:20.404283Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518647174405898:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:20.557744Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518647072285969:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.557911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518647072285970:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.476272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518648144717484:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57102
2025-11-11T17:19:20.449853Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518650818001123:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.457670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518650822955160:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.453922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518649886017564:2061] [node 4] ICP22 created new session: [1:7571518649886017585:2061]
2025-11-11T17:19:20.453905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518650818001074:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(260004215ns) {Configure:104(259995907ns) {ProcessConfigured:135(19448674ns) {StartInitialHandshake:157(19445888ns) {DropHandshakes:469(1910ns) {DropIncomingHandshake:441(361ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:476(2595ns) {SwitchToState:178(1058ns)}}}}}
2025-11-11T17:19:20.404410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518642879438524:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.404454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:20.404500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518642879438524:2048] [node 1] ICP22 created new session: [4:7571518647174405903:2048]
2025-11-11T17:19:20.404550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518647174405903:2048] [node 1] ICS09 handshake done sender: [4:7571518647174405898:2116] self: [4:7571518647174405901:0] peer: [1:7571518649886017572:0] socket: 275 qp: -1
2025-11-11T17:19:20.404578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518647174405903:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:20.404613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:20.421699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518642879438525:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.433945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518647174405906:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13912
2025-11-11T17:19:20.499726Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518647174405906:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.500038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518647174405908:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13912
2025-11-11T17:19:20.500126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518647174405909:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13912
2025-11-11T17:19:20.564427Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518647174405897:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:20.569993Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518647174405908:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.570083Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518647174405853:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.570101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518647174405853:2077] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.570933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518642879438526:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.577962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518647174405853:2077] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:20.579093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518642879438525:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.597676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.481884Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518648144717483:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.500398Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518650822955160:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.453995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518649886017585:2061] [node 4] ICS09 handshake done sender: [1:7571518649886017573:2114] self: [1:7571518649886017572:0] peer: [4:7571518647174405901:0] socket: 271 qp: -1
2025-11-11T17:19:20.465867Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518650818001122:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.500753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518650822955150:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.500789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:20.500829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518650822955150:2064] [node 4] ICP22 created new session: [2:7571518650822955171:2064]
2025-11-11T17:19:20.500879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518650822955171:2064] [node 4] ICS09 handshake done sender: [2:7571518650822955160:2113] self: [2:7571518650822955159:0] peer: [4:7571518647174405905:0] socket: 269 qp: -1
2025-11-11T17:19:20.500912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518650822955171:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:20.500944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:20.501402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518650822955170:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31237
2025-11-11T17:19:20.505742Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955168:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.519088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518650822955175:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31237
2025-11-11T17:19:20.534970Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518650822955164:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:20.535174Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955170:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.537099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518646527987791:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.537139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:20.537174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518646527987791:2048] [node 1] ICP22 created new session: [2:7571518650822955177:2048]
2025-11-11T17:19:20.537207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518650822955177:2048] [node 1] ICS09 handshake done sender: [2:7571518650822955164:2114] self: [2:7571518650822955167:0] peer: [1:7571518649886017574:0] socket: 274 qp: -1
2025-11-11T17:19:20.537232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518650822955177:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:20.537257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:20.539729Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955158:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.539764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518650822955158:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.578092Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518650822955158:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:20.586001Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955154:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.586042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518650822955154:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.623143Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518650822955154:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:20.626412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518650822955144:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.626464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:20.626504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518650822955144:2061] [node 5] ICP22 created new session: [2:7571518650822955182:2061]
2025-11-11T17:19:20.626545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518650822955182:2061] [node 5] ICS09 handshake done sender: [2:7571518650822955158:2112] self: [2:7571518650822955157:0] peer: [5:7571518650818001137:0] socket: 268 qp: -1
2025-11-11T17:19:20.626575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518650822955182:2061] [node 5] ICS10 traffic start
2025-11-11T17:19:20.626611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.622798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518647174405914:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13912
2025-11-11T17:19:20.466228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001135:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.454031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518649886017585:2061] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.489961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518648144717479:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(102284635ns) {Configure:104(102280826ns) {ProcessConfigured:135(65289ns) {StartInitialHandshake:157(62851ns) {DropHandshakes:469(2650ns) {DropIncomingHandshake:441(528ns) DropOutgoingHandshake:456(334ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:476(2028ns) {SwitchToState:178(880ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.645910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285976:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.651006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.613825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518642879438525:2049] [node 2] ICP22 created new session: [4:7571518647174405913:2049]
2025-11-11T17:19:20.509751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518648144717484:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.534389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518643849750118:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.534472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518648144717497:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57102
2025-11-11T17:19:20.546020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518648144717497:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.546334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518643849750119:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.565873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518648144717500:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57102
2025-11-11T17:19:20.566353Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518648144717484:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:20.566517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518643849750118:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.566567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:20.566612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518643849750118:2048] [node 1] ICP22 created new session: [3:7571518648144717501:2048]
2025-11-11T17:19:20.566652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518648144717501:2048] [node 1] ICS09 handshake done sender: [3:7571518648144717484:2113] self: [3:7571518648144717498:0] peer: [1:7571518649886017576:0] socket: 281 qp: -1
2025-11-11T17:19:20.566684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518648144717501:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:20.566733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:20.588660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518648144717482:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57102
2025-11-11T17:19:20.589807Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518648144717486:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.589834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518648144717486:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.602025Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518648144717486:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:20.626089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518648144717482:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.626182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518648144717505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57102
2025-11-11T17:19:20.627303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518648144717480:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.627372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:20.627410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518648144717480:2061] [node 4] ICP22 created new session: [3:7571518648144717506:2061]
2025-11-11T17:19:20.627469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518648144717506:2061] [node 4] ICS09 handshake done sender: [3:7571518648144717486:2114] self: [3:7571518648144717485:0] peer: [4:7571518647174405912:0] socket: 292 qp: -1
2025-11-11T17:19:20.627497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518648144717506:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:20.627521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:20.628447Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518648144717497:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:20.628696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518643849750119:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.628751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:20.628782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518643849750119:2049] [node 2] ICP22 created new session: [3:7571518648144717510:2049]
2025-11-11T17:19:20.628818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518648144717510:2049] [node 2] ICS09 handshake done sender: [3:7571518648144717497:2118] self: [3:7571518648144717499:0] peer: [2:7571518650822955153:0] socket: 303 qp: -1
2025-11-11T17:19:20.628842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518648144717510:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:20.628871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:20.653933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518646527987792:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.454074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:20.622996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518647174405851:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.693375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.477702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518646523033760:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(295671863ns) {Configure:104(295665471ns) {ProcessConfigured:135(19420729ns) {StartInitialHandshake:157(19417971ns) {DropHandshakes:469(1731ns) {DropIncomingHandshake:441(659ns) DropOutgoingHandshake:456(418ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:476(4213ns) {SwitchToState:178(1566ns)}}}}}
2025-11-11T17:19:20.690882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:20.466857Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017575:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.466888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518649886017575:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.499280Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017567:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.501718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518649886017584:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8530
2025-11-11T17:19:20.518181Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017584:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.518316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518649886017587:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8530
2025-11-11T17:19:20.533732Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518649886017575:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:20.535743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017577:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.535763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518649886017577:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.536224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518645591050205:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.536275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:20.536315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518645591050205:2048] [node 2] ICP22 created new session: [1:7571518649886017591:2048]
2025-11-11T17:19:20.536358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518649886017591:2048] [node 2] ICS09 handshake done sender: [1:7571518649886017575:2115] self: [1:7571518649886017574:0] peer: [2:7571518650822955167:0] socket: 272 qp: -1
2025-11-11T17:19:20.536384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518649886017591:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:20.536418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:20.550061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518649886017566:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(343398009ns) {Configure:104(343392517ns) {ProcessConfigured:135(66733ns) {StartInitialHandshake:157(64060ns) {DropHandshakes:469(2718ns) {DropIncomingHandshake:441(622ns) DropOutgoingHandshake:456(386ns)} GenerateSessionVirtualId:988(1059ns) PrepareNewSessionHandshake:476(1360ns) {SwitchToState:178(491ns)}}}}}
2025-11-11T17:19:20.564464Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017587:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.574325Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518649886017577:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:20.577960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518645591050206:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.578004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:20.578046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518645591050206:2049] [node 3] ICP22 created new session: [1:7571518649886017598:2049]
2025-11-11T17:19:20.578085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518649886017598:2049] [node 3] ICS09 handshake done sender: [1:7571518649886017577:2116] self: [1:7571518649886017576:0] peer: [3:7571518648144717498:0] socket: 273 qp: -1
2025-11-11T17:19:20.586074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518649886017598:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:20.586152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:20.588169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518645591050206:2049] [node 3] ICP18 handshake (actor [1:7571518649886017587:2118]) is too old
2025-11-11T17:19:20.605670Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017571:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.605712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518649886017571:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.614041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518649886017564:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(55952523ns) {HandleHandshakeStatus:329(55948102ns) {DropIncomingHandshake:441(16086ns) DropOutgoingHandshake:456(474ns) SwitchToState:178(433ns) ProcessPendingSessionEvents:495(55515127ns) {ForwardSessionEventToSession:612(19149788ns) ForwardSessionEventToSession:612(5793ns) ForwardSessionEventToSession:612(3640741ns) ForwardSessionEventToSession:612(20745583ns) ForwardSessionEventToSession:612(5243ns) ForwardSessionEventToSession:612(4212866ns) ForwardSessionEventToSession:612(3229ns) ForwardSessionEventToSession:612(7722925ns) ForwardSessionEventToSession:612(4489ns) ForwardSessionEventToSession:612(3286ns)}}}
2025-11-11T17:19:20.634037Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518649886017571:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:20.658344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518649886017565:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.658403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:20.658444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518649886017565:2063] [node 5] ICP22 created new session: [1:7571518649886017601:2063]
2025-11-11T17:19:20.658484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518649886017601:2063] [node 5] ICS09 handshake done sender: [1:7571518649886017571:2113] self: [1:7571518649886017570:0] peer: [5:7571518650818001140:0] socket: 300 qp: -1
2025-11-11T17:19:20.658513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518649886017601:2063] [node 5] ICS10 traffic start
2025-11-11T17:19:20.658543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.509634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518646523033760:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.519347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001138:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.572404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.578254Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518650818001138:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.578746Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518650818001123:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:20.578933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518646523033759:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.578975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518646523033760:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.589652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:20.605629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518646523033760:2049] [node 2] ICP22 created new session: [5:7571518650818001141:2049]
2025-11-11T17:19:20.637652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518650818001141:2049] [node 2] ICS09 handshake done sender: [5:7571518650818001123:2112] self: [5:7571518650818001137:0] peer: [2:7571518650822955157:0] socket: 279 qp: -1
2025-11-11T17:19:20.637724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518650818001135:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.641801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518650818001141:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:20.641849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:20.659994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518650818001077:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.660374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001144:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.690362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518646523033760:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(111149623ns) {HandleHandshakeStatus:329(111148496ns) {DropIncomingHandshake:441(236ns) DropOutgoingHandshake:456(3622ns) SwitchToState:178(70ns) ProcessPendingSessionEvents:495(13787227ns) {ForwardSessionEventToSession:612(13736712ns) ForwardSessionEventToSession:612(4859ns) ForwardSessionEventToSession:612(3126ns) ForwardSessionEventToSession:612(8644ns) ForwardSessionEventToSession:612(5559ns) ForwardSessionEventToSession:612(4779ns) ForwardSessionEventToSession:612(2677ns) ForwardSessionEventToSession:612(2191ns) ForwardSessionEventToSession:612(5884ns) ForwardSessionEventToSession:612(1460ns)}}}
2025-11-11T17:19:20.690475Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21025) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:20.709812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518646523033759:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21025) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518650818001138:2118] held: no
2025-11-11T17:19:20.711497Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:20.712253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001145:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.712390Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518650818001138:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:20.719129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518650818001077:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518650818001135:2117] held: no
2025-11-11T17:19:20.719259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518646523033759:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.719389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:20.719429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518646523033759:2048] [node 1] ICP22 created new session: [5:7571518650818001148:2048]
2025-11-11T17:19:20.719467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518650818001148:2048] [node 1] ICS09 handshake done sender: [5:7571518650818001138:2118] self: [5:7571518650818001140:0] peer: [1:7571518649886017570:0] socket: 302 qp: -1
2025-11-11T17:19:20.719495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518650818001148:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:20.719524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:20.720263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001147:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.720613Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518650818001145:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.720730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518646523033761:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.726279Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518650818001135:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.637643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:20.698465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518646527987792:2049] [node 3] ICP22 created new session: [2:7571518650822955184:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.729884Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518647072285976:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.729973Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518647072285979:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.730164Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518647072285969:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:20.761709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518647072285970:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.761807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:20.761851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518647072285970:2070] [node 4] ICP22 created new session: [6:7571518647072285981:2070]
2025-11-11T17:19:20.761902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518647072285981:2070] [node 4] ICS09 handshake done sender: [6:7571518647072285969:2113] self: [6:7571518647072285975:0] peer: [4:7571518647174405852:0] socket: 276 qp: -1
2025-11-11T17:19:20.761933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518647072285981:2070] [node 4] ICS10 traffic start
2025-11-11T17:19:20.766133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518650818001077:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.766182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:20.766220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518650818001077:2062] [node 4] ICP22 created new session: [5:7571518650818001152:2062]
2025-11-11T17:19:20.766261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518650818001152:2062] [node 4] ICS09 handshake done sender: [5:7571518650818001135:2117] self: [5:7571518650818001146:0] peer: [4:7571518647174405895:0] socket: 290 qp: -1
2025-11-11T17:19:20.766288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518650818001152:2062] [node 4] ICS10 traffic start
2025-11-11T17:19:20.766338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:20.641692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518647174405851:2064] [node 6] ICP22 created new session: [4:7571518647174405915:2064]
2025-11-11T17:19:20.750077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518648144717512:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57102
2025-11-11T17:19:20.761967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:20.721110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518650822955184:2049] [node 3] ICS09 handshake done sender: [2:7571518650822955154:2110] self: [2:7571518650822955153:0] peer: [3:7571518648144717499:0] socket: 298 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.761226Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518648144717496:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.761276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518648144717496:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.794152Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518648144717496:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:20.794323Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518648144717490:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.794354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518648144717490:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.777642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518649886017595:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.789967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518650818001154:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36118
2025-11-11T17:19:20.762492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285980:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.649735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518647174405913:2049] [node 2] ICS09 handshake done sender: [4:7571518647174405897:2115] self: [4:7571518647174405905:0] peer: [2:7571518650822955159:0] socket: 270 qp: -1
2025-11-11T17:19:20.806777Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518650818001145:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:20.721147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518650822955184:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:20.817622Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518650818001129:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.817674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518650818001129:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.651617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518647174405913:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:20.653982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518647174405915:2064] [node 6] ICS09 handshake done sender: [4:7571518647174405853:2077] self: [4:7571518647174405852:0] peer: [6:7571518647072285975:0] socket: 262 qp: -1
2025-11-11T17:19:20.826420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518646523033761:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.826478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:20.826525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518646523033761:2050] [node 3] ICP22 created new session: [5:7571518650818001155:2050]
2025-11-11T17:19:20.826566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518650818001155:2050] [node 3] ICS09 handshake done sender: [5:7571518650818001145:2121] self: [5:7571518650818001151:0] peer: [3:7571518648144717495:0] socket: 317 qp: -1
2025-11-11T17:19:20.826601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518650818001155:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:20.826632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:20.833999Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518650818001129:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:20.834929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518650818001074:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.834989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:20.835040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518650818001074:2065] [node 6] ICP22 created new session: [5:7571518650818001158:2065]
2025-11-11T17:19:20.835078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518650818001158:2065] [node 6] ICS09 handshake done sender: [5:7571518650818001129:2115] self: [5:7571518650818001126:0] peer: [6:7571518647072285987:0] socket: 316 qp: -1
2025-11-11T17:19:20.835108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518650818001158:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:20.835142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.777699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518649886017595:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.762605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.733727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:20.659195Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3807) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:20.661689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518647174405915:2064] [node 6] ICS10 traffic start
2025-11-11T17:19:20.665776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:20.666238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518642879438525:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(86957396ns) {HandleHandshakeStatus:329(86956165ns) {DropIncomingHandshake:441(342ns) DropOutgoingHandshake:456(4924ns) SwitchToState:178(161ns) ProcessPendingSessionEvents:495(111284ns) {ForwardSessionEventToSession:612(56692ns) ForwardSessionEventToSession:612(5932ns) ForwardSessionEventToSession:612(3477ns) ForwardSessionEventToSession:612(3350ns) ForwardSessionEventToSession:612(6255ns) ForwardSessionEventToSession:612(3177ns) ForwardSessionEventToSession:612(3056ns) ForwardSessionEventToSession:612(7344ns) ForwardSessionEventToSession:612(5284ns) ForwardSessionEventToSession:612(9201ns)}}}
2025-11-11T17:19:20.666325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518642879438526:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3807) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518647174405908:2120] held: no
2025-11-11T17:19:20.666526Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518647174405896:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.666546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518647174405896:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.667471Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518647174405908:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:20.669870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:20.678171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518647174405851:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(60236018ns) {HandleHandshakeStatus:329(60233713ns) {DropIncomingHandshake:441(1529ns) DropOutgoingHandshake:456(650ns) SwitchToState:178(458ns) ProcessPendingSessionEvents:495(101856ns) {ForwardSessionEventToSession:612(56683ns) ForwardSessionEventToSession:612(2136ns) ForwardSessionEventToSession:612(2281ns) ForwardSessionEventToSession:612(4384ns) ForwardSessionEventToSession:612(2503ns) ForwardSessionEventToSession:612(3161ns) ForwardSessionEventToSession:612(10208ns) ForwardSessionEventToSession:612(2650ns) ForwardSessionEventToSession:612(5149ns) ForwardSessionEventToSession:612(2639ns)}}}
2025-11-11T17:19:20.690737Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518647174405896:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:20.690967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518642879438526:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.694927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518647174405894:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.694976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:20.695015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518647174405894:2062] [node 5] ICP22 created new session: [4:7571518647174405919:2062]
2025-11-11T17:19:20.695057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518647174405919:2062] [node 5] ICS09 handshake done sender: [4:7571518647174405896:2114] self: [4:7571518647174405895:0] peer: [5:7571518650818001146:0] socket: 289 qp: -1
2025-11-11T17:19:20.695090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518647174405919:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:20.695122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:20.713808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:20.713883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518642879438526:2050] [node 3] ICP22 created new session: [4:7571518647174405921:2050]
2025-11-11T17:19:20.713932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518647174405921:2050] [node 3] ICS09 handshake done sender: [4:7571518647174405908:2120] self: [4:7571518647174405912:0] peer: [3:7571518648144717485:0] socket: 295 qp: -1
2025-11-11T17:19:20.713964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518647174405921:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:20.714007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:20.714215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518642879438526:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(23165383ns) {HandleHandshakeStatus:329(23163293ns) {DropIncomingHandshake:441(1070ns) DropOutgoingHandshake:456(388ns) SwitchToState:178(486ns) ProcessPendingSessionEvents:495(873ns)}}
2025-11-11T17:19:20.822560Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518648144717490:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:20.762713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518642777318600:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.797751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955166:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.797779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518650822955166:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:20.806019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518646527987792:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(127855130ns) {HandleHandshakeStatus:329(127852114ns) {DropIncomingHandshake:441(12280ns) DropOutgoingHandshake:456(413ns) SwitchToState:178(894ns) ProcessPendingSessionEvents:495(20457937ns) {ForwardSessionEventToSession:612(4436852ns) ForwardSessionEventToSession:612(3017ns) ForwardSessionEventToSession:612(5660ns) ForwardSessionEventToSession:612(3890360ns) ForwardSessionEventToSession:612(3028ns) ForwardSessionEventToSession:612(3537163ns) ForwardSessionEventToSession:612(3591ns) ForwardSessionEventToSession:612(2511ns) ForwardSessionEventToSession:612(8558410ns) ForwardSessionEventToSession:612(3537ns)}}}
2025-11-11T17:19:20.821995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518650822955186:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31237
2025-11-11T17:19:20.825725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518650822955166:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:20.840733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518650822955186:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.854322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518650822955112:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.854384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:20.854424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518650822955112:2059] [node 6] ICP22 created new session: [2:7571518650822955189:2059]
2025-11-11T17:19:20.854478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518650822955189:2059] [node 6] ICS09 handshake done sender: [2:7571518650822955166:2115] self: [2:7571518650822955165:0] peer: [6:7571518647072285984:0] socket: 309 qp: -1
2025-11-11T17:19:20.854509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518650822955189:2059] [node 6] ICS10 traffic start
2025-11-11T17:19:20.854532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:20.778607Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518649886017595:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:20.834374Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:20.846910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518648144717479:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.846995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:20.847044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518648144717479:2060] [node 5] ICP22 created new session: [3:7571518648144717515:2060]
2025-11-11T17:19:20.847087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518648144717515:2060] [node 5] ICS09 handshake done sender: [3:7571518648144717496:2117] self: [3:7571518648144717495:0] peer: [5:7571518650818001151:0] socket: 265 qp: -1
2025-11-11T17:19:20.847120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518648144717515:2060] [node 5] ICS10 traffic start
2025-11-11T17:19:20.847155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:20.847388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518648144717479:2060] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:20.847736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518648144717481:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.847819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:20.847863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518648144717481:2071] [node 6] ICP22 created new session: [3:7571518648144717517:2071]
2025-11-11T17:19:20.847894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518648144717517:2071] [node 6] ICS09 handshake done sender: [3:7571518648144717490:2116] self: [3:7571518648144717489:0] peer: [6:7571518647072285986:0] socket: 293 qp: -1
2025-11-11T17:19:20.847921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518648144717517:2071] [node 6] ICS10 traffic start
2025-11-11T17:19:20.847970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:20.778865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518649886017566:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:20.762767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518642777318601:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.778935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:20.778980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518649886017566:2062] [node 6] ICP22 created new session: [1:7571518649886017604:2062]
2025-11-11T17:19:20.779021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518649886017604:2062] [node 6] ICS09 handshake done sender: [1:7571518649886017595:2119] self: [1:7571518649886017594:0] peer: [6:7571518647072285983:0] socket: 304 qp: -1
2025-11-11T17:19:20.779052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518649886017604:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:20.779084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:20.770635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518647072285977:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.770709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518647072285980:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:20.772160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518642777318602:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.772243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518642777318617:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:20.829813Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21025) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:20.837917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518642777318600:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21025) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518647072285976:2116] held: no
2025-11-11T17:19:20.838120Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:20.839139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285989:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.839472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285990:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.839785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285991:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.848700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518642777318617:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518647072285980:2120] held: no
2025-11-11T17:19:20.854172Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518647072285977:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:20.855468Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518647072285979:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:20.855598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518642777318602:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.855656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:20.855707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518642777318602:2050] [node 3] ICP22 created new session: [6:7571518647072285992:2050]
2025-11-11T17:19:20.855743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518647072285992:2050] [node 3] ICS09 handshake done sender: [6:7571518647072285977:2117] self: [6:7571518647072285986:0] peer: [3:7571518648144717489:0] socket: 311 qp: -1
2025-11-11T17:19:20.855784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518647072285992:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:20.855838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:20.856270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518642777318601:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.856315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:20.856349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518642777318601:2049] [node 2] ICP22 created new session: [6:7571518647072285994:2049]
2025-11-11T17:19:20.856390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518647072285994:2049] [node 2] ICS09 handshake done sender: [6:7571518647072285979:2119] self: [6:7571518647072285984:0] peer: [2:7571518650822955165:0] socket: 291 qp: -1
2025-11-11T17:19:20.856410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518647072285994:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:20.856443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:20.879767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518647072285988:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59953
2025-11-11T17:19:20.894095Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518647072285976:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:20.895454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518642777318617:2061] [node 5] ICP35 event processing took too much time PendingConnection:278(20166454ns) {HandleHandshakeStatus:395(20162652ns) {LogHandshakeFail:472(11600ns) {UpdateErrorStateLog:544(3964ns)} DropOutgoingHandshake:456(1389ns) ProcessPendingSessionEvents:495(12917531ns) {DropSessionEvent:512(9247ns) DropSessionEvent:512(4340ns) DropSessionEvent:512(4155ns) DropSessionEvent:512(9010922ns) DropSessionEvent:512(4418ns) DropSessionEvent:512(3855527ns) DropSessionEvent:512(10044ns) DropSessionEvent:512(3896ns) DropSessionEvent:512(4206ns) DropSessionEvent:512(4205ns)}}}
2025-11-11T17:19:20.895583Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518647072285980:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:20.895882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518642777318617:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.895944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:20.895986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518642777318617:2061] [node 5] ICP22 created new session: [6:7571518647072285996:2061]
2025-11-11T17:19:20.896025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518647072285996:2061] [node 5] ICS09 handshake done sender: [6:7571518647072285980:2120] self: [6:7571518647072285987:0] peer: [5:7571518650818001126:0] socket: 263 qp: -1
2025-11-11T17:19:20.896056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518647072285996:2061] [node 5] ICS10 traffic start
2025-11-11T17:19:20.896093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:20.896657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518642777318600:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:20.896710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:20.896754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518642777318600:2048] [node 1] ICP22 created new session: [6:7571518647072285998:2048]
2025-11-11T17:19:20.896790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518647072285998:2048] [node 1] ICS09 handshake done sender: [6:7571518647072285976:2116] self: [6:7571518647072285983:0] peer: [1:7571518649886017594:0] socket: 306 qp: -1
2025-11-11T17:19:20.896816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518647072285998:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:20.896844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:20.953444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:20.953660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518650822955177:2048] [node 1] ICS01 socket: 274 reason# EndOfStream
2025-11-11T17:19:20.953700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518646527987791:2048] [node 1] ICP30 unregister session Session# [2:7571518650822955177:2048] VirtualId# [2:7571518650822955167:0]
2025-11-11T17:19:20.953736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518650822955177:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:20.953817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:20.953853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:20.966133Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:20.969671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:20.966345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518647174405903:2048] [node 1] ICS01 socket: 275 reason# EndOfStream
2025-11-11T17:19:20.966377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518642879438524:2048] [node 1] ICP30 unregister session Session# [4:7571518647174405903:2048] VirtualId# [4:7571518647174405901:0]
2025-11-11T17:19:20.966402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518647174405903:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:20.966457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:20.969667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:20.970812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518647072285998:2048] [node 1] ICS01 socket: 306 reason# EndOfStream
2025-11-11T17:19:20.970866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518642777318600:2048] [node 1] ICP30 unregister session Session# [6:7571518647072285998:2048] VirtualId# [6:7571518647072285983:0]
2025-11-11T17:19:20.970899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518647072285998:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:20.970968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:20.971004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:20.965900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:20.966158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518648144717501:2048] [node 1] ICS01 socket: 281 reason# EndOfStream
2025-11-11T17:19:20.966194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518643849750118:2048] [node 1] ICP30 unregister session Session# [3:7571518648144717501:2048] VirtualId# [3:7571518648144717498:0]
2025-11-11T17:19:20.966230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518648144717501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:20.966304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:20.966340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:20.975321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:20.975523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518650818001148:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-11-11T17:19:20.975554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518646523033759:2048] [node 1] ICP30 unregister session Session# [5:7571518650818001148:2048] VirtualId# [5:7571518650818001140:0]
2025-11-11T17:19:20.975579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518650818001148:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:20.975632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:20.975663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:21.113767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:21.114004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518648144717510:2049] [node 2] ICS01 socket: 303 reason# EndOfStream
2025-11-11T17:19:21.114045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518643849750119:2049] [node 2] ICP30 unregister session Session# [3:7571518648144717510:2049] VirtualId# [3:7571518648144717499:0]
2025-11-11T17:19:21.114082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518648144717510:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.114178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:21.114219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:21.121752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:21.121761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:21.121961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518647174405913:2049] [node 2] ICS01 socket: 270 reason# EndOfStream
2025-11-11T17:19:21.122368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518647072285994:2049] [node 2] ICS01 socket: 291 reason# EndOfStream
2025-11-11T17:19:21.121999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518642879438525:2049] [node 2] ICP30 unregister session Session# [4:7571518647174405913:2049] VirtualId# [4:7571518647174405905:0]
2025-11-11T17:19:21.122398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518642777318601:2049] [node 2] ICP30 unregister session Session# [6:7571518647072285994:2049] VirtualId# [6:7571518647072285984:0]
2025-11-11T17:19:21.122421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518647072285994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.122031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518647174405913:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.122475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:21.122101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:21.122506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:21.122133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:21.121825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:21.122018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518650818001141:2049] [node 2] ICS07 socket disconnect 279 reason# EndOfStream
2025-11-11T17:19:21.122043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518650818001141:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.122100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:21.122137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518650818001141:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:21.149754Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1402) Couldn't connect to any resolved address
2025-11-11T17:19:21.150036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518646523033760:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1402) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:21.150092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518650818001141:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:21.150123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518646523033760:2049] [node 2] ICP30 unregister session Session# [5:7571518650818001141:2049] VirtualId# [5:7571518650818001137:0]
2025-11-11T17:19:21.150168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:21.150201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518646523033760:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:21.150213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:21.165690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518646523033760:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:21.221778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:21.222008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518650818001155:2050] [node 3] ICS01 socket: 317 reason# EndOfStream
2025-11-11T17:19:21.222044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518646523033761:2050] [node 3] ICP30 unregister session Session# [5:7571518650818001155:2050] VirtualId# [5:7571518650818001151:0]
2025-11-11T17:19:21.222081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518650818001155:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.222154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:21.222192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:21.222267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:21.222443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518647072285992:2050] [node 3] ICS01 socket: 311 reason# EndOfStream
2025-11-11T17:19:21.222468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518642777318602:2050] [node 3] ICP30 unregister session Session# [6:7571518647072285992:2050] VirtualId# [6:7571518647072285986:0]
2025-11-11T17:19:21.222486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518647072285992:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.222521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:21.222546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:21.225730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:21.226014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518647174405921:2050] [node 3] ICS01 socket: 295 reason# EndOfStream
2025-11-11T17:19:21.226072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518642879438526:2050] [node 3] ICP30 unregister session Session# [4:7571518647174405921:2050] VirtualId# [4:7571518647174405912:0]
2025-11-11T17:19:21.226116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518647174405921:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.226221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:21.226285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:21.281768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:21.281999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518650818001152:2062] [node 4] ICS01 socket: 290 reason# EndOfStream
2025-11-11T17:19:21.282037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518650818001077:2062] [node 4] ICP30 unregister session Session# [5:7571518650818001152:2062] VirtualId# [5:7571518650818001146:0]
2025-11-11T17:19:21.282068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518650818001152:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.282152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:21.282186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:21.282275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:21.282459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518647072285981:2070] [node 4] ICS01 socket: 276 reason# EndOfStream
2025-11-11T17:19:21.282496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518647072285970:2070] [node 4] ICP30 unregister session Session# [6:7571518647072285981:2070] VirtualId# [6:7571518647072285975:0]
2025-11-11T17:19:21.282519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518647072285981:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.282557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:21.282586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:21.337726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:21.337966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518647072285996:2061] [node 5] ICS01 socket: 263 reason# EndOfStream
2025-11-11T17:19:21.338008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518642777318617:2061] [node 5] ICP30 unregister session Session# [6:7571518647072285996:2061] VirtualId# [6:7571518647072285987:0]
2025-11-11T17:19:21.338039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518647072285996:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:21.338128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:21.338165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:21.501673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518652820626607:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:21.501782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518652820626608:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:21.556733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518651513857812:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:21.556778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518651513857813:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:21.602051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518653750146749:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:21.602095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518653750146750:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:21.661633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518655507450538:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:21.661679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518655507450539:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:21.661774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518655507450540:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:21.694065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518653998352478:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:21.694109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518653998352479:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:21.697845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518653998352477:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:21.909990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518651391966887:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:21.910036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518651391966888:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:21.910060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518651391966889:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:22.081852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518659802417851:2060] [node 6] ICP01 ready to work
2025-11-11T17:19:22.077045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518657115593959:2102] [node 6] ICP01 ready to work
2025-11-11T17:19:22.141948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518655686934203:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:22.135946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518658293319801:2062] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.213424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518658293319804:2064] [node 6] ICP01 ready to work
2025-11-11T17:19:22.246235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518657115593968:2100] [node 5] ICP01 ready to work
2025-11-11T17:19:22.230296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518655686934218:2075] [node 5] ICP01 ready to work
2025-11-11T17:19:22.249738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518655808825171:2075] [node 6] ICP01 ready to work
2025-11-11T17:19:22.246129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518658045114109:2069] [node 5] ICP01 ready to work
2025-11-11T17:19:22.249803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518659802417875:2062] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.246527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518658045114110:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:22.265669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518655808825170:2074] [node 4] ICP01 ready to work
2025-11-11T17:19:22.299847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319819:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.353622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518657115593969:2103] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.234996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518658045114107:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:22.277884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518655808825172:2073] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.428588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518655686934218:2075] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(182112155ns) {Configure:104(182107724ns) {ProcessConfigured:135(52598ns) {StartInitialHandshake:157(50989ns) {DropHandshakes:469(2053ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(420ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:476(1836ns) {SwitchToState:178(643ns)}}}}}
2025-11-11T17:19:22.379736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319844:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.357971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518657115593959:2102] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(280668510ns) {Configure:104(280663965ns) {ProcessConfigured:135(57266ns) {StartInitialHandshake:157(54954ns) {DropHandshakes:469(2509ns) {DropIncomingHandshake:441(913ns) DropOutgoingHandshake:456(430ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:476(1446ns) {SwitchToState:178(583ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.429851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518655507450540:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(237334186ns) {Configure:104(237329460ns) {ProcessConfigured:135(65265ns) {StartInitialHandshake:157(63293ns) {DropHandshakes:469(1993ns) {DropIncomingHandshake:441(452ns) DropOutgoingHandshake:456(454ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:476(3673ns) {SwitchToState:178(501ns)}}}}}
2025-11-11T17:19:22.378752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518655808825174:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36202
2025-11-11T17:19:22.271779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518658045114111:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44294
2025-11-11T17:19:22.395969Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518658293319819:2087] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.359669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518657115593970:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4893
2025-11-11T17:19:22.436716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518655507450538:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(255303049ns) {Configure:104(255296816ns) {ProcessConfigured:135(58195ns) {StartInitialHandshake:157(55415ns) {DropHandshakes:469(2571ns) {DropIncomingHandshake:441(519ns) DropOutgoingHandshake:456(652ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:476(1422ns) {SwitchToState:178(535ns)}}}}}
2025-11-11T17:19:22.437264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417901:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
2025-11-11T17:19:22.469674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518659802417901:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.475171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518655507450538:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.378914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518655808825175:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36202
2025-11-11T17:19:22.417968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319845:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.464115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934258:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.386212Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593970:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.279381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518653750146749:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(97301683ns) {Configure:104(97296499ns) {ProcessConfigured:135(53245ns) {StartInitialHandshake:157(47784ns) {DropHandshakes:469(2915ns) {DropIncomingHandshake:441(601ns) DropOutgoingHandshake:456(356ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:476(1517ns) {SwitchToState:178(537ns)}}}}}
2025-11-11T17:19:22.379923Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825174:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.470703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518651391966888:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(268869641ns) {Configure:104(268859068ns) {ProcessConfigured:135(58453ns) {StartInitialHandshake:157(56354ns) {DropHandshakes:469(2529ns) {DropIncomingHandshake:441(771ns) DropOutgoingHandshake:456(446ns)} GenerateSessionVirtualId:988(642ns) PrepareNewSessionHandshake:476(1559ns) {SwitchToState:178(539ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.482455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518658293319801:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(301329683ns) {Configure:104(301325125ns) {ProcessConfigured:135(49117ns) {StartInitialHandshake:157(47789ns) {DropHandshakes:469(1958ns) {DropIncomingHandshake:441(506ns) DropOutgoingHandshake:456(378ns)} GenerateSessionVirtualId:988(741ns) PrepareNewSessionHandshake:476(1236ns) {SwitchToState:178(408ns)}}}}}
2025-11-11T17:19:22.410744Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114111:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.506368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518659802417851:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(408398856ns) {Configure:104(381017697ns) {ProcessConfigured:135(8411681ns) {StartInitialHandshake:157(8408785ns) {DropHandshakes:469(3150ns) {DropIncomingHandshake:441(1081ns) DropOutgoingHandshake:456(431ns)} GenerateSessionVirtualId:988(799ns) PrepareNewSessionHandshake:476(3027ns) {SwitchToState:178(1510ns)}}}}}
2025-11-11T17:19:22.507863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417912:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.386718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518657115593971:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4893
2025-11-11T17:19:22.462285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518658045114124:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44294
2025-11-11T17:19:22.477939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518658045114107:2065] [node 6] ICP35 event processing took too much time PendingConnection:278(16051890ns) {IncomingHandshake:261(16049666ns) {DropIncomingHandshake:441(668ns) PrepareNewSessionHandshake:476(11782998ns) {SwitchToState:178(5073ns)} GenerateSessionVirtualId:988(736ns) IssueIncomingHandshakeReply:193(6479ns)}}
2025-11-11T17:19:22.555176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518658045114125:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44294
2025-11-11T17:19:22.561652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114124:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.513814Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518655686934258:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.459453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518655808825172:2073] [node 5] ICP35 event processing took too much time PendingConnection:278(21085013ns) {IncomingHandshake:261(37954ns) {DropIncomingHandshake:441(637ns) PrepareNewSessionHandshake:476(2629ns) {SwitchToState:178(925ns)} GenerateSessionVirtualId:988(305ns) IssueIncomingHandshakeReply:193(6100ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.482584Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518658293319845:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.580529Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518659802417912:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.513947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518651391966887:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.387012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518657115593981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4893
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.482995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518653998352477:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.565758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825175:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.569692Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114125:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.601456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417913:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.594100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934264:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.597684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934257:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.414586Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518652820626608:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(339239543ns) {Configure:104(332601217ns) {ProcessConfigured:135(67267ns) {StartInitialHandshake:157(62127ns) {DropHandshakes:469(2773ns) {DropIncomingHandshake:441(927ns) DropOutgoingHandshake:456(438ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:476(1665ns) {SwitchToState:178(597ns)}}}}}
2025-11-11T17:19:22.443901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593971:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.444517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518657115593987:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4893
2025-11-11T17:19:22.485820Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593975:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.485844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518657115593975:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.486635Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518657115593975:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:22.486969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518657115593968:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.487041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:22.487083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518657115593968:2100] [node 5] ICP22 created new session: [1:7571518657115593990:2100]
2025-11-11T17:19:22.487119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518657115593990:2100] [node 5] ICS09 handshake done sender: [1:7571518657115593975:2114] self: [1:7571518657115593974:0] peer: [5:7571518658293319851:0] socket: 281 qp: -1
2025-11-11T17:19:22.487150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518657115593990:2100] [node 5] ICS10 traffic start
2025-11-11T17:19:22.487180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:22.505529Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593979:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.505571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518657115593979:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.513669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [1:7571518657115593968:2100] [node 5] ICP35 event processing took too much time StateWork:293(16290725ns)
2025-11-11T17:19:22.555437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593977:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.555474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518657115593977:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.566831Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593987:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.575000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593981:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.601134Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518657115593979:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:22.610069Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593986:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.610096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518657115593986:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.610767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518657115593969:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.610821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:22.610866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518657115593969:2103] [node 4] ICP22 created new session: [1:7571518657115593996:2103]
2025-11-11T17:19:22.610926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518657115593996:2103] [node 4] ICS09 handshake done sender: [1:7571518657115593979:2116] self: [1:7571518657115593978:0] peer: [4:7571518659802417908:0] socket: 282 qp: -1
2025-11-11T17:19:22.610956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518657115593996:2103] [node 4] ICS10 traffic start
2025-11-11T17:19:22.610991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:22.483072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319850:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.525750Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.534500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518658293319804:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(256467345ns) {Configure:104(256462801ns) {ProcessConfigured:135(57515ns) {StartInitialHandshake:157(54623ns) {DropHandshakes:469(2675ns) {DropIncomingHandshake:441(543ns) DropOutgoingHandshake:456(588ns)} GenerateSessionVirtualId:988(725ns) PrepareNewSessionHandshake:476(1297ns) {SwitchToState:178(473ns)}}}}}
2025-11-11T17:19:22.536053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518653998352477:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518658293319845:2114] held: no
2025-11-11T17:19:22.536387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319853:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.566262Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518658293319845:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:22.573932Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518658293319850:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.574194Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518658293319844:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.574537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518658293319801:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.574607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518653998352478:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.599555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319846:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.606668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518653998352477:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.606718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:22.612244Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518658293319846:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.613636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518653998352477:2048] [node 1] ICP22 created new session: [5:7571518658293319861:2048]
2025-11-11T17:19:22.624615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319863:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.625625Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518658293319844:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.626848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518655686934257:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.627022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518651391966889:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.634563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934266:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.638559Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518655686934258:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:22.594549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518653750146749:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.612714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825179:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.637728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518653998352479:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(488894274ns) {Configure:104(488889110ns) {ProcessConfigured:135(48821679ns) {StartInitialHandshake:157(48819680ns) {DropHandshakes:469(2192ns) {DropIncomingHandshake:441(396ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(678ns) PrepareNewSessionHandshake:476(2237ns) {SwitchToState:178(939ns)}}}}}
2025-11-11T17:19:22.645698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518658293319861:2048] [node 1] ICS09 handshake done sender: [5:7571518658293319845:2114] self: [5:7571518658293319851:0] peer: [1:7571518657115593974:0] socket: 283 qp: -1
2025-11-11T17:19:22.646335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518653998352479:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.646644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319864:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.647222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518653998352478:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.614487Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518659802417907:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.653666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518658293319861:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:22.653754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:22.653799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518653998352478:2049] [node 2] ICP22 created new session: [5:7571518658293319866:2049]
2025-11-11T17:19:22.630692Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518657115593986:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.612756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518655808825179:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.653845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518658293319866:2049] [node 2] ICS09 handshake done sender: [5:7571518658293319844:2112] self: [5:7571518658293319860:0] peer: [2:7571518655808825178:0] socket: 279 qp: -1
2025-11-11T17:19:22.653872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518658293319866:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:22.653904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:22.655509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:22.638777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518651391966887:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.614823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518658045114110:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(339305819ns) {Configure:104(339301452ns) {ProcessConfigured:135(7686310ns) {StartInitialHandshake:157(7684270ns) {DropHandshakes:469(679ns) {DropIncomingHandshake:441(164ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(2577ns) {SwitchToState:178(1305ns)}}}}}
2025-11-11T17:19:22.623635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518655808825185:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36202
2025-11-11T17:19:22.625106Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518655808825179:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:22.635312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518655808825170:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(238966155ns) {Configure:104(238960805ns) {ProcessConfigured:135(4690654ns) {StartInitialHandshake:157(4688248ns) {DropHandshakes:469(4272ns) {DropIncomingHandshake:441(443ns) DropOutgoingHandshake:456(348ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:476(2771ns) {SwitchToState:178(1143ns)}}}}}
2025-11-11T17:19:22.650224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518655808825172:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.638082Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518657115593977:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:22.614520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518659802417907:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.638826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:22.615365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518658045114129:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44294
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.650285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.634125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518655507450539:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(356162173ns) {Configure:104(356157684ns) {ProcessConfigured:135(46198ns) {StartInitialHandshake:157(44591ns) {DropHandshakes:469(1619ns) {DropIncomingHandshake:441(229ns) DropOutgoingHandshake:456(433ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:476(521ns) {SwitchToState:178(103ns)}}}}}
2025-11-11T17:19:22.652964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518652820626608:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.638866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518651391966887:2048] [node 1] ICP22 created new session: [6:7571518655686934267:2048]
2025-11-11T17:19:22.624909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518658045114131:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44294
2025-11-11T17:19:22.650336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518655808825172:2073] [node 5] ICP22 created new session: [2:7571518655808825191:2073]
2025-11-11T17:19:22.653007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.647069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518659802417875:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(17832574ns) {IncomingHandshake:261(17831355ns) {DropIncomingHandshake:441(124ns) PrepareNewSessionHandshake:476(17806307ns) {SwitchToState:178(1508ns)} GenerateSessionVirtualId:988(683ns) IssueIncomingHandshakeReply:193(5325ns)}}
2025-11-11T17:19:22.638909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518655686934267:2048] [node 1] ICS09 handshake done sender: [6:7571518655686934258:2114] self: [6:7571518655686934261:0] peer: [1:7571518657115593976:0] socket: 293 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.709948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518653998352477:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(92445374ns) {HandleHandshakeStatus:329(92443495ns) {DropIncomingHandshake:441(1840ns) DropOutgoingHandshake:456(414ns) SwitchToState:178(404ns) ProcessPendingSessionEvents:495(1195ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.626772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518653750146749:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(30775822ns) {IncomingHandshake:261(19338199ns) {DropIncomingHandshake:441(558ns) PrepareNewSessionHandshake:476(548ns) {SwitchToState:178(195ns)} GenerateSessionVirtualId:988(199ns) IssueIncomingHandshakeReply:193(11324595ns) {SwitchToState:178(4788ns)}}}
2025-11-11T17:19:22.662233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518657115593959:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.661995Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518659802417901:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:22.638935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518655686934267:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:22.650386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518655808825191:2073] [node 5] ICS09 handshake done sender: [2:7571518655808825179:2113] self: [2:7571518655808825178:0] peer: [5:7571518658293319860:0] socket: 278 qp: -1
2025-11-11T17:19:22.710063Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12473) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.669933Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518659802417907:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:22.662279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:22.646836Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114119:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.646869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518658045114119:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.686184Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114117:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.686210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518658045114117:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.687142Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518658045114119:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:22.687285Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518658045114125:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:22.689770Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114131:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.693230Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518658045114117:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:22.650423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518655808825191:2073] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.638961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:22.713638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518658293319856:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.694369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518653750146749:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.662330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518657115593959:2102] [node 6] ICP22 created new session: [1:7571518657115594002:2102]
2025-11-11T17:19:22.671576Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.650462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:22.645841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518655686934203:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(418352313ns) {Configure:104(418346936ns) {ProcessConfigured:135(15476148ns) {StartInitialHandshake:157(15473587ns) {DropHandshakes:469(2135ns) {DropIncomingHandshake:441(539ns) DropOutgoingHandshake:456(365ns)} GenerateSessionVirtualId:988(858ns) PrepareNewSessionHandshake:476(1893ns) {SwitchToState:178(817ns)}}}}}
2025-11-11T17:19:22.672288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518655507450538:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518659802417901:2112] held: no
2025-11-11T17:19:22.694443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:22.713687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518658293319856:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.727264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518658293319801:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12473) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518658293319850:2117] held: no
2025-11-11T17:19:22.729873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518658293319850:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:22.742868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518658293319801:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.742916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:22.742959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518658293319801:2062] [node 4] ICP22 created new session: [5:7571518658293319872:2062]
2025-11-11T17:19:22.743008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518658293319872:2062] [node 4] ICS09 handshake done sender: [5:7571518658293319850:2117] self: [5:7571518658293319859:0] peer: [4:7571518659802417906:0] socket: 295 qp: -1
2025-11-11T17:19:22.743038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518658293319872:2062] [node 4] ICS10 traffic start
2025-11-11T17:19:22.743092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:22.765716Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518658293319856:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:22.766314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518658293319804:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.766364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:22.766426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518658293319804:2064] [node 6] ICP22 created new session: [5:7571518658293319874:2064]
2025-11-11T17:19:22.766479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518658293319874:2064] [node 6] ICS09 handshake done sender: [5:7571518658293319856:2119] self: [5:7571518658293319855:0] peer: [6:7571518655686934271:0] socket: 265 qp: -1
2025-11-11T17:19:22.766509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518658293319874:2064] [node 6] ICS10 traffic start
2025-11-11T17:19:22.766541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:22.771539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518658293319869:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49210
2025-11-11T17:19:22.818097Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518658293319819:2087] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:22.826084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518653998352479:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.662377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518657115594002:2102] [node 6] ICS09 handshake done sender: [1:7571518657115593977:2115] self: [1:7571518657115593976:0] peer: [6:7571518655686934261:0] socket: 287 qp: -1
2025-11-11T17:19:22.687750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518659802417875:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.687810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:22.687874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518659802417875:2062] [node 5] ICP22 created new session: [4:7571518659802417918:2062]
2025-11-11T17:19:22.687924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518659802417918:2062] [node 5] ICS09 handshake done sender: [4:7571518659802417907:2113] self: [4:7571518659802417906:0] peer: [5:7571518658293319859:0] socket: 294 qp: -1
2025-11-11T17:19:22.687957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518659802417918:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:22.687993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:22.689700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518655507450538:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.689742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:22.689781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518655507450538:2048] [node 1] ICP22 created new session: [4:7571518659802417921:2048]
2025-11-11T17:19:22.701829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518659802417921:2048] [node 1] ICS09 handshake done sender: [4:7571518659802417901:2112] self: [4:7571518659802417908:0] peer: [1:7571518657115593978:0] socket: 288 qp: -1
2025-11-11T17:19:22.701873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518659802417921:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:22.701915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:22.744502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417924:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
2025-11-11T17:19:22.744556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417925:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
2025-11-11T17:19:22.744598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417926:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
2025-11-11T17:19:22.793833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518659802417925:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.797449Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518659802417926:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.801795Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518659802417924:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.810214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518655507450540:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.821669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518655507450539:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.831969Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518659802417911:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.832006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518659802417911:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.833887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518655507450539:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(26752135ns) {IncomingHandshake:261(26750559ns) {DropIncomingHandshake:441(304ns) PrepareNewSessionHandshake:476(599ns) {SwitchToState:178(188ns)} GenerateSessionVirtualId:988(193ns) IssueIncomingHandshakeReply:193(11913718ns) {SwitchToState:178(3942ns)}}}
2025-11-11T17:19:22.851145Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518659802417911:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.673989Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518655686934264:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.650943Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825185:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.694489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518653750146749:2048] [node 1] ICP22 created new session: [3:7571518658045114134:2048]
2025-11-11T17:19:22.850997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:22.662404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518657115594002:2102] [node 6] ICS10 traffic start
2025-11-11T17:19:22.689062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518655686934218:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.694532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518658045114134:2048] [node 1] ICS09 handshake done sender: [3:7571518658045114125:2116] self: [3:7571518658045114126:0] peer: [1:7571518657115593985:0] socket: 299 qp: -1
2025-11-11T17:19:22.706165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934270:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.734075Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518655686934257:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:22.734579Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.735806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518651391966889:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518655686934257:2113] held: no
2025-11-11T17:19:22.735902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518651391966889:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.735941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:22.735979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518651391966889:2050] [node 3] ICP22 created new session: [6:7571518655686934274:2050]
2025-11-11T17:19:22.736020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518655686934274:2050] [node 3] ICS09 handshake done sender: [6:7571518655686934257:2113] self: [6:7571518655686934265:0] peer: [3:7571518658045114118:0] socket: 284 qp: -1
2025-11-11T17:19:22.736050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518655686934274:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:22.736079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:22.736427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934272:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.756279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934277:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.756401Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518655686934272:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.756521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518651391966888:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.761623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934273:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.770605Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.774057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518655686934264:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:22.789674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518655686934218:2075] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518655686934264:2117] held: no
2025-11-11T17:19:22.793855Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518655686934273:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.794151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518655686934203:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.794228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934279:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.818991Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518655686934272:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:22.856867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518655686934218:2075] [node 5] ICP35 event processing took too much time PendingConnection:278(27381922ns) {HandleHandshakeStatus:395(27379091ns) {LogHandshakeFail:472(11964ns) {UpdateErrorStateLog:544(4067ns)} DropOutgoingHandshake:456(1171ns) ProcessPendingSessionEvents:495(20002118ns) {DropSessionEvent:512(16078ns) DropSessionEvent:512(5659ns) DropSessionEvent:512(4481ns) DropSessionEvent:512(5559ns) DropSessionEvent:512(4622ns) DropSessionEvent:512(4705ns) DropSessionEvent:512(12970ns) DropSessionEvent:512(7307983ns) DropSessionEvent:512(7465800ns) DropSessionEvent:512(20241ns)}}}
2025-11-11T17:19:22.858163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518651391966888:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.865730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518655686934218:2075] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.873732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:22.873766Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.875073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518655686934218:2075] [node 5] ICP22 created new session: [6:7571518655686934281:2075]
2025-11-11T17:19:22.882055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518655686934281:2075] [node 5] ICS09 handshake done sender: [6:7571518655686934264:2117] self: [6:7571518655686934271:0] peer: [5:7571518658293319855:0] socket: 301 qp: -1
2025-11-11T17:19:22.882101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518655686934281:2075] [node 5] ICS10 traffic start
2025-11-11T17:19:22.882137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:22.882367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518655686934218:2075] [node 5] ICP35 event processing took too much time PendingConnection:278(25346024ns) {HandleHandshakeStatus:329(25343895ns) {DropIncomingHandshake:441(2967ns) DropOutgoingHandshake:456(417ns) SwitchToState:178(556ns) ProcessPendingSessionEvents:495(1120ns)}}
2025-11-11T17:19:22.883834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518655686934282:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5933
2025-11-11T17:19:22.884055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:22.884104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518651391966888:2049] [node 2] ICP22 created new session: [6:7571518655686934284:2049]
2025-11-11T17:19:22.856516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417931:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
2025-11-11T17:19:22.857727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518653998352479:2050] [node 3] ICP22 created new session: [5:7571518658293319876:2050]
2025-11-11T17:19:22.662430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:22.665865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518655808825171:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(267724325ns) {Configure:104(267719376ns) {ProcessConfigured:135(22688394ns) {StartInitialHandshake:157(22684811ns) {DropHandshakes:469(5206788ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(1878ns)} GenerateSessionVirtualId:988(1461ns) PrepareNewSessionHandshake:476(2599ns) {SwitchToState:178(629ns)}}}}}
2025-11-11T17:19:22.891814Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518655686934273:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:22.893665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518655686934284:2049] [node 2] ICS09 handshake done sender: [6:7571518655686934272:2120] self: [6:7571518655686934278:0] peer: [2:7571518655808825190:0] socket: 290 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:22.874994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518659802417925:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:22.881939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518658293319876:2050] [node 3] ICS09 handshake done sender: [5:7571518658293319819:2087] self: [5:7571518658293319865:0] peer: [3:7571518658045114116:0] socket: 272 qp: -1
2025-11-11T17:19:22.666483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518651513857812:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.694565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518658045114134:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:22.666568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518655808825193:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36202
2025-11-11T17:19:22.672500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518651513857813:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(407180525ns) {Configure:104(407174102ns) {ProcessConfigured:135(18277522ns) {StartInitialHandshake:157(18275732ns) {DropHandshakes:469(2161ns) {DropIncomingHandshake:441(545ns) DropOutgoingHandshake:456(468ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:476(1844ns) {SwitchToState:178(706ns)}}}}}
2025-11-11T17:19:22.690857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518655808825198:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36202
2025-11-11T17:19:22.718418Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825184:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.718440Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518655808825184:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.719148Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518655808825185:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:22.721227Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.722166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518651513857812:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.722231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:22.722282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518651513857812:2048] [node 1] ICP22 created new session: [2:7571518655808825201:2048]
2025-11-11T17:19:22.722323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518655808825201:2048] [node 1] ICS09 handshake done sender: [2:7571518655808825185:2115] self: [2:7571518655808825195:0] peer: [1:7571518657115593972:0] socket: 307 qp: -1
2025-11-11T17:19:22.722353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518655808825201:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:22.722393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:22.722638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518651513857812:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:22.729922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825193:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.734378Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518655808825184:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:22.749801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518651513857813:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.749858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:22.749901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518651513857813:2049] [node 3] ICP22 created new session: [2:7571518655808825204:2049]
2025-11-11T17:19:22.749939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518655808825204:2049] [node 3] ICS09 handshake done sender: [2:7571518655808825184:2114] self: [2:7571518655808825183:0] peer: [3:7571518658045114137:0] socket: 309 qp: -1
2025-11-11T17:19:22.749973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518655808825204:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:22.753638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:22.782274Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825194:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.782308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518655808825194:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.786480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518651513857813:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(36506930ns) {HandleHandshakeStatus:329(36504526ns) {DropIncomingHandshake:441(13250ns) DropOutgoingHandshake:456(445ns) SwitchToState:178(557ns) ProcessPendingSessionEvents:495(12359037ns) {ForwardSessionEventToSession:612(12304487ns) ForwardSessionEventToSession:612(9576ns) ForwardSessionEventToSession:612(3257ns) ForwardSessionEventToSession:612(2443ns) ForwardSessionEventToSession:612(4845ns) ForwardSessionEventToSession:612(2920ns) ForwardSessionEventToSession:612(3195ns) ForwardSessionEventToSession:612(10678ns) ForwardSessionEventToSession:612(2935ns) ForwardSessionEventToSession:612(4515ns)}}}
2025-11-11T17:19:22.801948Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518655808825194:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:22.826207Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825189:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.826235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518655808825189:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.833697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518655808825171:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.843010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:22.843055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518655808825171:2075] [node 6] ICP22 created new session: [2:7571518655808825209:2075]
2025-11-11T17:19:22.850944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518655808825189:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:22.862661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518655808825199:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36202
2025-11-11T17:19:22.862735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518655808825209:2075] [node 6] ICS09 handshake done sender: [2:7571518655808825194:2118] self: [2:7571518655808825190:0] peer: [6:7571518655686934278:0] socket: 273 qp: -1
2025-11-11T17:19:22.866581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518655808825170:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.866642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:22.866698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518655808825170:2074] [node 4] ICP22 created new session: [2:7571518655808825210:2074]
2025-11-11T17:19:22.866741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518655808825210:2074] [node 4] ICS09 handshake done sender: [2:7571518655808825189:2116] self: [2:7571518655808825188:0] peer: [4:7571518659802417927:0] socket: 277 qp: -1
2025-11-11T17:19:22.866777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518655808825210:2074] [node 4] ICS10 traffic start
2025-11-11T17:19:22.866835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:22.881786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518655808825209:2075] [node 6] ICS10 traffic start
2025-11-11T17:19:22.881846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:22.890471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518655808825171:2075] [node 6] ICP35 event processing took too much time PendingConnection:278(64078748ns) {HandleHandshakeStatus:329(56614685ns) {DropIncomingHandshake:441(19577ns) DropOutgoingHandshake:456(510ns) SwitchToState:178(549ns) ProcessPendingSessionEvents:495(106557ns) {ForwardSessionEventToSession:612(54136ns) ForwardSessionEventToSession:612(8185ns) ForwardSessionEventToSession:612(7506ns) ForwardSessionEventToSession:612(2084ns) ForwardSessionEventToSession:612(2448ns) ForwardSessionEventToSession:612(4648ns) ForwardSessionEventToSession:612(3340ns) ForwardSessionEventToSession:612(4911ns) ForwardSessionEventToSession:612(3132ns) ForwardSessionEventToSession:612(5350ns)}}}
2025-11-11T17:19:22.897735Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518655808825199:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.917685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518655808825170:2074] [node 4] ICP18 handshake (actor [2:7571518655808825199:2120]) is too old
2025-11-11T17:19:22.667265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518652820626608:2049] [node 3] ICP22 created new session: [1:7571518657115594001:2049]
2025-11-11T17:19:22.694597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:22.905725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518655686934284:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:22.884463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518658293319876:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:22.886848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518659802417851:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.884512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:22.938093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518653998352479:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(111745240ns) {HandleHandshakeStatus:329(111742351ns) {DropIncomingHandshake:441(662ns) DropOutgoingHandshake:456(11628ns) SwitchToState:178(852ns) ProcessPendingSessionEvents:495(23740877ns) {ForwardSessionEventToSession:612(23686815ns) ForwardSessionEventToSession:612(6660ns) ForwardSessionEventToSession:612(8217ns) ForwardSessionEventToSession:612(5467ns) ForwardSessionEventToSession:612(3103ns) ForwardSessionEventToSession:612(5935ns) ForwardSessionEventToSession:612(3906ns) ForwardSessionEventToSession:612(2887ns) ForwardSessionEventToSession:612(5682ns) ForwardSessionEventToSession:612(2150ns)}}}
2025-11-11T17:19:22.913725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:22.886906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:22.677733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518657115594001:2049] [node 3] ICS09 handshake done sender: [1:7571518657115593986:2118] self: [1:7571518657115593985:0] peer: [3:7571518658045114126:0] socket: 269 qp: -1
2025-11-11T17:19:22.698262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518653750146750:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:22.930110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518655686934203:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.930204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:22.930280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518655686934203:2064] [node 4] ICP22 created new session: [6:7571518655686934286:2064]
2025-11-11T17:19:22.930329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518655686934286:2064] [node 4] ICS09 handshake done sender: [6:7571518655686934273:2121] self: [6:7571518655686934280:0] peer: [4:7571518659802417910:0] socket: 267 qp: -1
2025-11-11T17:19:22.930370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518655686934286:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:22.930423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:22.931455Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12473) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.958306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518651391966888:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(124360271ns) {HandleHandshakeStatus:329(112433399ns) {DropIncomingHandshake:441(1821ns) DropOutgoingHandshake:456(5626ns) SwitchToState:178(572ns) ProcessPendingSessionEvents:495(18359783ns) {ForwardSessionEventToSession:612(18308729ns) ForwardSessionEventToSession:612(5748ns) ForwardSessionEventToSession:612(3833ns) ForwardSessionEventToSession:612(6018ns) ForwardSessionEventToSession:612(3065ns) ForwardSessionEventToSession:612(7850ns) ForwardSessionEventToSession:612(3387ns) ForwardSessionEventToSession:612(3687ns) ForwardSessionEventToSession:612(5386ns) ForwardSessionEventToSession:612(2744ns)}}}
2025-11-11T17:19:22.958385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518651391966888:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:22.958930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518655686934203:2064] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:22.681643Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518657115593973:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.886963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518659802417851:2060] [node 6] ICP22 created new session: [4:7571518659802417933:2060]
2025-11-11T17:19:22.887011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518659802417933:2060] [node 6] ICS09 handshake done sender: [4:7571518659802417911:2114] self: [4:7571518659802417910:0] peer: [6:7571518655686934280:0] socket: 320 qp: -1
2025-11-11T17:19:22.887043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518659802417933:2060] [node 6] ICS10 traffic start
2025-11-11T17:19:22.887093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:22.887475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518655507450539:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.887517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:22.887559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518655507450539:2049] [node 2] ICP22 created new session: [4:7571518659802417935:2049]
2025-11-11T17:19:22.887592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518659802417935:2049] [node 2] ICS09 handshake done sender: [4:7571518659802417925:2119] self: [4:7571518659802417927:0] peer: [2:7571518655808825188:0] socket: 326 qp: -1
2025-11-11T17:19:22.887628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518659802417935:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:22.887660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:22.902585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518659802417932:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47408
2025-11-11T17:19:22.950412Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518659802417924:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:22.969848Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.971545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518655507450540:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518659802417924:2118] held: no
2025-11-11T17:19:22.714354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518658045114107:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.714443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:22.714497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518658045114107:2065] [node 6] ICP22 created new session: [3:7571518658045114138:2065]
2025-11-11T17:19:22.714540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518658045114138:2065] [node 6] ICS09 handshake done sender: [3:7571518658045114119:2114] self: [3:7571518658045114118:0] peer: [6:7571518655686934265:0] socket: 275 qp: -1
2025-11-11T17:19:22.714577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518658045114138:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:22.714609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:22.714952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518658045114109:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.714979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:22.715012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518658045114109:2069] [node 5] ICP22 created new session: [3:7571518658045114140:2069]
2025-11-11T17:19:22.715050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518658045114140:2069] [node 5] ICS09 handshake done sender: [3:7571518658045114117:2113] self: [3:7571518658045114116:0] peer: [5:7571518658293319865:0] socket: 271 qp: -1
2025-11-11T17:19:22.715074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518658045114140:2069] [node 5] ICS10 traffic start
2025-11-11T17:19:22.715100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:22.750486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518658045114142:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44294
2025-11-11T17:19:22.756752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518653750146750:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16402273ns) {IncomingHandshake:261(11263222ns) {DropIncomingHandshake:441(310ns) PrepareNewSessionHandshake:476(2051ns) {SwitchToState:178(457ns)} GenerateSessionVirtualId:988(442ns) IssueIncomingHandshakeReply:193(11238109ns) {SwitchToState:178(3837ns)}}}
2025-11-11T17:19:22.777665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518658045114143:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44294
2025-11-11T17:19:22.789848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518658045114131:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:22.809705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518653750146750:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.821649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114143:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.821739Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:22.825815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:22.825864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518653750146750:2049] [node 2] ICP22 created new session: [3:7571518658045114144:2049]
2025-11-11T17:19:22.825926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518658045114144:2049] [node 2] ICS09 handshake done sender: [3:7571518658045114131:2119] self: [3:7571518658045114137:0] peer: [2:7571518655808825183:0] socket: 311 qp: -1
2025-11-11T17:19:22.842869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518658045114144:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:22.842924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:22.846301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518658045114109:2069] [node 5] ICP18 handshake (actor [3:7571518658045114143:2121]) is too old
2025-11-11T17:19:22.850736Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518658045114128:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:22.850774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518658045114128:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.877833Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518658045114128:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:22.883166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518653750146750:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(73292438ns) {HandleHandshakeStatus:329(73290067ns) {DropIncomingHandshake:441(735ns) DropOutgoingHandshake:456(4915ns) SwitchToState:178(578ns) ProcessPendingSessionEvents:495(9273093ns) {ForwardSessionEventToSession:612(9219087ns) ForwardSessionEventToSession:612(6745ns) ForwardSessionEventToSession:612(3803ns) ForwardSessionEventToSession:612(9693ns) ForwardSessionEventToSession:612(2075ns) ForwardSessionEventToSession:612(3886ns) ForwardSessionEventToSession:612(2185ns) ForwardSessionEventToSession:612(2882ns) ForwardSessionEventToSession:612(4737ns) ForwardSessionEventToSession:612(2790ns)}}}
2025-11-11T17:19:22.883228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518653750146750:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:22.914297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518658045114110:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.914372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:22.914421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518658045114110:2064] [node 4] ICP22 created new session: [3:7571518658045114147:2064]
2025-11-11T17:19:22.914473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518658045114147:2064] [node 4] ICS09 handshake done sender: [3:7571518658045114128:2117] self: [3:7571518658045114127:0] peer: [4:7571518659802417929:0] socket: 322 qp: -1
2025-11-11T17:19:22.914505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518658045114147:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:22.914534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:22.681678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518657115593973:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:22.973681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518655507450540:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:22.689354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518657115594001:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:22.689404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:22.697906Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518657115593973:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:22.719340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518652820626607:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:22.719392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:22.719439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518652820626607:2048] [node 2] ICP22 created new session: [1:7571518657115594006:2048]
2025-11-11T17:19:22.719478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518657115594006:2048] [node 2] ICS09 handshake done sender: [1:7571518657115593973:2113] self: [1:7571518657115593972:0] peer: [2:7571518655808825195:0] socket: 305 qp: -1
2025-11-11T17:19:22.719513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518657115594006:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:22.719542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:22.821962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518652820626608:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(129454943ns) {HandleHandshakeStatus:329(129453001ns) {DropIncomingHandshake:441(573ns) DropOutgoingHandshake:456(458ns) SwitchToState:178(454ns) ProcessPendingSessionEvents:495(68471601ns) {ForwardSessionEventToSession:612(27698905ns) ForwardSessionEventToSession:612(4221401ns) ForwardSessionEventToSession:612(5332ns) ForwardSessionEventToSession:612(7899088ns) ForwardSessionEventToSession:612(4259463ns) ForwardSessionEventToSession:612(3559ns) ForwardSessionEventToSession:612(3909ns) ForwardSessionEventToSession:612(11229498ns) ForwardSessionEventToSession:612(2748ns) ForwardSessionEventToSession:612(10706138ns)}}}
2025-11-11T17:19:22.989676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:22.989799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518655507450540:2050] [node 3] ICP22 created new session: [4:7571518659802417937:2050]
2025-11-11T17:19:22.989852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518659802417937:2050] [node 3] ICS09 handshake done sender: [4:7571518659802417924:2118] self: [4:7571518659802417929:0] peer: [3:7571518658045114127:0] socket: 280 qp: -1
2025-11-11T17:19:22.989905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518659802417937:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:22.989968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:22.990222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518655507450540:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(16458846ns) {HandleHandshakeStatus:329(16457331ns) {DropIncomingHandshake:441(1061ns) DropOutgoingHandshake:456(264ns) SwitchToState:178(452ns) ProcessPendingSessionEvents:495(1218ns)}}
2025-11-11T17:19:23.047166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:23.047375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518655808825201:2048] [node 1] ICS07 socket disconnect 307 reason# EndOfStream
2025-11-11T17:19:23.047403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518655808825201:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.047466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:23.047498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518655808825201:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:23.055454Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:19:23.057536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:23.057770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518658045114134:2048] [node 1] ICS01 socket: 299 reason# EndOfStream
2025-11-11T17:19:23.057808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518653750146749:2048] [node 1] ICP30 unregister session Session# [3:7571518658045114134:2048] VirtualId# [3:7571518658045114126:0]
2025-11-11T17:19:23.057845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518658045114134:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.057934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:23.057984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:23.057747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518651513857812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:19:23.060463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518655808825201:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:23.065540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:23.065818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518659802417921:2048] [node 1] ICS01 socket: 288 reason# EndOfStream
2025-11-11T17:19:23.065869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518655507450538:2048] [node 1] ICP30 unregister session Session# [4:7571518659802417921:2048] VirtualId# [4:7571518659802417908:0]
2025-11-11T17:19:23.065919Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Couldn't connect to any resolved address
2025-11-11T17:19:23.065914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518659802417921:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.066023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:23.066067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:23.066399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:23.066597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518655686934267:2048] [node 1] ICS01 socket: 293 reason# EndOfStream
2025-11-11T17:19:23.066639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518651391966887:2048] [node 1] ICP30 unregister session Session# [6:7571518655686934267:2048] VirtualId# [6:7571518655686934261:0]
2025-11-11T17:19:23.066663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518655686934267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.066730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:23.066767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:23.067159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518651513857812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:23.067423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518655808825201:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:23.067466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518651513857812:2048] [node 1] ICP30 unregister session Session# [2:7571518655808825201:2048] VirtualId# [2:7571518655808825195:0]
2025-11-11T17:19:23.067511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:23.067545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518651513857812:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.067570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7443) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.069691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:23.069910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518658293319861:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-11-11T17:19:23.070059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518653998352477:2048] [node 1] ICP30 unregister session Session# [5:7571518658293319861:2048] VirtualId# [5:7571518658293319851:0]
2025-11-11T17:19:23.070100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518658293319861:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.070189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:23.070229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:23.081727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518651513857812:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:23.151423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:23.151650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518658045114144:2049] [node 2] ICS07 socket disconnect 311 reason# EndOfStream
2025-11-11T17:19:23.151692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518658045114144:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.151765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:23.151803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518658045114144:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:23.154642Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:19:23.155325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518653750146750:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:19:23.155361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518658045114144:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:23.155945Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address
2025-11-11T17:19:23.156119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518653750146750:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:23.156166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518658045114144:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:23.156195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518653750146750:2049] [node 2] ICP30 unregister session Session# [3:7571518658045114144:2049] VirtualId# [3:7571518658045114137:0]
2025-11-11T17:19:23.156236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:23.156267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518653750146750:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.156281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.165784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:23.166013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518659802417935:2049] [node 2] ICS01 socket: 326 reason# EndOfStream
2025-11-11T17:19:23.166058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518655507450539:2049] [node 2] ICP30 unregister session Session# [4:7571518659802417935:2049] VirtualId# [4:7571518659802417927:0]
2025-11-11T17:19:23.166094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518659802417935:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.166220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:23.166275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:23.166248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:23.166437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518655686934284:2049] [node 2] ICS07 socket disconnect 290 reason# EndOfStream
2025-11-11T17:19:23.166462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518655686934284:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.166517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:23.166555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518655686934284:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:23.165747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:23.165979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518658293319866:2049] [node 2] ICS01 socket: 279 reason# EndOfStream
2025-11-11T17:19:23.166020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518653998352478:2049] [node 2] ICP30 unregister session Session# [5:7571518658293319866:2049] VirtualId# [5:7571518658293319860:0]
2025-11-11T17:19:23.166051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518658293319866:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.166127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:23.166167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:23.167560Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address
2025-11-11T17:19:23.167769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518651391966888:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:23.167805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518655686934284:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:23.167847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518651391966888:2049] [node 2] ICP30 unregister session Session# [6:7571518655686934284:2049] VirtualId# [6:7571518655686934278:0]
2025-11-11T17:19:23.167881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:23.167920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518651391966888:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.167949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.176161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518653750146750:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:23.184913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518651391966888:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:23.301300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:23.301530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518655686934274:2050] [node 3] ICS01 socket: 284 reason# EndOfStream
2025-11-11T17:19:23.301574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518651391966889:2050] [node 3] ICP30 unregister session Session# [6:7571518655686934274:2050] VirtualId# [6:7571518655686934265:0]
2025-11-11T17:19:23.301617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518655686934274:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.301690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:23.301731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:23.301809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:23.302003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:23.301996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518658293319876:2050] [node 3] ICS07 socket disconnect 272 reason# EndOfStream
2025-11-11T17:19:23.302225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518659802417937:2050] [node 3] ICS01 socket: 280 reason# EndOfStream
2025-11-11T17:19:23.302033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518658293319876:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.302266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518655507450540:2050] [node 3] ICP30 unregister session Session# [4:7571518659802417937:2050] VirtualId# [4:7571518659802417929:0]
2025-11-11T17:19:23.302074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:23.302105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518658293319876:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:23.302302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518659802417937:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.302424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:23.302468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:23.302971Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Couldn't connect to any resolved address
2025-11-11T17:19:23.303195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518653998352479:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:23.303234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518658293319876:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:23.303262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518653998352479:2050] [node 3] ICP30 unregister session Session# [5:7571518658293319876:2050] VirtualId# [5:7571518658293319865:0]
2025-11-11T17:19:23.303302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:23.303328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518653998352479:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.303341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1709) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:23.317700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518653998352479:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:23.397774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:23.398014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518655686934286:2064] [node 4] ICS01 socket: 267 reason# EndOfStream
2025-11-11T17:19:23.398063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518655686934203:2064] [node 4] ICP30 unregister session Session# [6:7571518655686934286:2064] VirtualId# [6:7571518655686934280:0]
2025-11-11T17:19:23.398102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518655686934286:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.397956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:23.398221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:23.398203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518658293319872:2062] [node 4] ICS01 socket: 295 reason# EndOfStream
2025-11-11T17:19:23.398279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:23.398242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518658293319801:2062] [node 4] ICP30 unregister session Session# [5:7571518658293319872:2062] VirtualId# [5:7571518658293319859:0]
2025-11-11T17:19:23.398272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518658293319872:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:23.398347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:23.398391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:23.708672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518662331970150:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:23.708719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518662331970151:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:23.765830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518663850783037:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:23.765871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518663850783038:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:23.845914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518662333640423:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:23.845966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518662333640424:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:23.886056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518661769494599:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:23.886102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518661769494600:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:23.886127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518661769494601:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:23.933972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518663305052184:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:23.934013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518663305052185:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:23.934040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518663305052186:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:23.990164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518659987298115:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:23.990210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518659987298116:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:23.990235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518659987298117:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:24.219873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518664282265424:2059] [node 4] ICP01 ready to work
2025-11-11T17:19:24.229144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518666064461911:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.315872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518666626937505:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-11T17:19:24.242207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518668145750363:2069] [node 4] ICP01 ready to work
2025-11-11T17:19:24.366743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518666628607741:2069] [node 4] ICP01 ready to work
2025-11-11T17:19:24.353078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518666064461922:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:24.373714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518667600019500:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.353073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518666626937510:2080] [node 6] ICP01 ready to work
2025-11-11T17:19:24.374624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518668145750395:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50526
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.445119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518666628607771:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6255
2025-11-11T17:19:24.445251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518666628607772:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6255
2025-11-11T17:19:24.462785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518659987298116:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(125440681ns) {Configure:104(125436604ns) {ProcessConfigured:135(52064ns) {StartInitialHandshake:157(49937ns) {DropHandshakes:469(2097ns) {DropIncomingHandshake:441(421ns) DropOutgoingHandshake:456(362ns)} GenerateSessionVirtualId:988(487ns) PrepareNewSessionHandshake:476(1462ns) {SwitchToState:178(740ns)}}}}}
2025-11-11T17:19:24.466813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607771:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.466899Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607772:2097] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.489903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518662333640423:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(263313746ns) {Configure:104(263309427ns) {ProcessConfigured:135(35328939ns) {StartInitialHandshake:157(35325887ns) {DropHandshakes:469(1906ns) {DropIncomingHandshake:441(462ns) DropOutgoingHandshake:456(322ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:476(3355ns) {SwitchToState:178(1847ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.422459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518666064461911:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(193043504ns) {Configure:104(193039794ns) {ProcessConfigured:135(53812ns) {StartInitialHandshake:157(51503ns) {DropHandshakes:469(2200ns) {DropIncomingHandshake:441(533ns) DropOutgoingHandshake:456(506ns)} GenerateSessionVirtualId:988(1048ns) PrepareNewSessionHandshake:476(881ns) {SwitchToState:178(333ns)}}}}}
2025-11-11T17:19:24.508344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518664282265451:2082] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.397676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937505:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.437633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518668145750398:2098] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.525344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518661769494600:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(179396913ns) {Configure:104(179393548ns) {ProcessConfigured:135(48510ns) {StartInitialHandshake:157(45936ns) {DropHandshakes:469(2609ns) {DropIncomingHandshake:441(768ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:476(1296ns) {SwitchToState:178(505ns)}}}}}
2025-11-11T17:19:24.431328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518667600019507:2059] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.585630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518666628607786:2064] [node 6] ICP01 ready to work
2025-11-11T17:19:24.433623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518666626937513:2081] [node 5] ICP01 ready to work
2025-11-11T17:19:24.437660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518668145750400:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.600151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518664282265482:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:24.549683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019514:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:24.546803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518666064461958:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17737
2025-11-11T17:19:24.461640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518666626937515:2082] [node 4] ICP01 ready to work
2025-11-11T17:19:24.613775Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607743:2070] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.445805Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750395:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.550595Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518666064461958:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.550709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518661769494601:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.621933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518666064461972:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17737
2025-11-11T17:19:24.641953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518666064461973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17737
2025-11-11T17:19:24.493526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518666626937516:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-11T17:19:24.650022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518664282265482:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.580182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019546:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:24.585676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518667600019514:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.610033Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518667600019546:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.670677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518667600019537:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.670719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518667600019537:2099] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.701799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518667600019537:2099] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.613814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518666628607743:2070] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.650167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518664282265451:2082] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.712559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518664282265487:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:24.578595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518668145750410:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50526
2025-11-11T17:19:24.559251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937516:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.613742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518666628607793:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:24.647452Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518666628607743:2070] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:24.654568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518666628607741:2069] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.654617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:24.654663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518666628607741:2069] [node 4] ICP22 created new session: [3:7571518666628607797:2069]
2025-11-11T17:19:24.654724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518666628607797:2069] [node 4] ICS09 handshake done sender: [3:7571518666628607743:2070] self: [3:7571518666628607742:0] peer: [4:7571518666064461961:0] socket: 277 qp: -1
2025-11-11T17:19:24.654756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518666628607797:2069] [node 4] ICS10 traffic start
2025-11-11T17:19:24.654791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:24.673914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518662333640424:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(354228957ns) {Configure:104(326002702ns) {ProcessConfigured:135(61706ns) {StartInitialHandshake:157(59773ns) {DropHandshakes:469(2097ns) {DropIncomingHandshake:441(517ns) DropOutgoingHandshake:456(363ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:476(1248ns) {SwitchToState:178(405ns)}}}}}
2025-11-11T17:19:24.705820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518662333640424:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.706269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518666628607800:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6255
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.731147Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518666064461972:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.711060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518666626937513:2081] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(149194856ns) {Configure:104(149191770ns) {ProcessConfigured:135(94829ns) {StartInitialHandshake:157(51240ns) {DropHandshakes:469(2011ns) {DropIncomingHandshake:441(381ns) DropOutgoingHandshake:456(384ns)} GenerateSessionVirtualId:988(3884ns) PrepareNewSessionHandshake:476(882ns) {SwitchToState:178(316ns)}} IncomingHandshake:261(18927ns) {DropIncomingHandshake:441(209ns) PrepareNewSessionHandshake:476(236ns) {SwitchToState:178(111ns)} GenerateSessionVirtualId:988(123ns) IssueIncomingHandshakeReply:193(678ns)}}}}
2025-11-11T17:19:24.731591Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518666064461958:2103] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:24.711170Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937507:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.578651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518668145750411:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50526
2025-11-11T17:19:24.761764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518666064461950:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17737
2025-11-11T17:19:24.711187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518666626937507:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.757986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518659987298115:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(432019493ns) {Configure:104(432014747ns) {ProcessConfigured:135(7919141ns) {StartInitialHandshake:157(7916392ns) {DropHandshakes:469(718ns) {DropIncomingHandshake:441(197ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(2521ns) {SwitchToState:178(1395ns)}}}}}
2025-11-11T17:19:24.761971Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518664282265482:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:24.741654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518666628607805:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6255
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.771564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518661769494601:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.771613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:24.771688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518661769494601:2050] [node 3] ICP22 created new session: [4:7571518666064461979:2050]
2025-11-11T17:19:24.771720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518666064461979:2050] [node 3] ICS09 handshake done sender: [4:7571518666064461958:2103] self: [4:7571518666064461961:0] peer: [3:7571518666628607742:0] socket: 285 qp: -1
2025-11-11T17:19:24.771745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518666064461979:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:24.771776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:24.772600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518666064461978:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17737
2025-11-11T17:19:24.590189Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750411:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.749689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518663305052185:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(351648145ns) {Configure:104(351643170ns) {ProcessConfigured:135(30820977ns) {StartInitialHandshake:157(30819050ns) {DropHandshakes:469(2444ns) {DropIncomingHandshake:441(432ns) DropOutgoingHandshake:456(504ns)} GenerateSessionVirtualId:988(668ns) PrepareNewSessionHandshake:476(2457ns) {SwitchToState:178(1144ns)}}}}}
2025-11-11T17:19:24.720409Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518666626937507:2107] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:24.750471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518666628607741:2069] [node 4] ICP35 event processing took too much time PendingConnection:278(95706772ns) {HandleHandshakeStatus:329(95703674ns) {DropIncomingHandshake:441(10065ns) DropOutgoingHandshake:456(382ns) SwitchToState:178(218ns) ProcessPendingSessionEvents:495(20530548ns) {ForwardSessionEventToSession:612(20482686ns) ForwardSessionEventToSession:612(5805ns) ForwardSessionEventToSession:612(3611ns) ForwardSessionEventToSession:612(5046ns) ForwardSessionEventToSession:612(2807ns) ForwardSessionEventToSession:612(5055ns) ForwardSessionEventToSession:612(5274ns) ForwardSessionEventToSession:612(2501ns) ForwardSessionEventToSession:612(4156ns) ForwardSessionEventToSession:612(1761ns)}}}
2025-11-11T17:19:24.600731Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750410:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.750075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518663305052186:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(316492555ns) {Configure:104(316488646ns) {ProcessConfigured:135(21319899ns) {StartInitialHandshake:157(21316890ns) {DropHandshakes:469(1437ns) {DropIncomingHandshake:441(272ns) DropOutgoingHandshake:456(375ns)} GenerateSessionVirtualId:988(334ns) PrepareNewSessionHandshake:476(3926ns) {SwitchToState:178(2085ns)}}}}}
2025-11-11T17:19:24.769742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518667600019507:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.797748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:24.801956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518667600019500:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(406966749ns) {Configure:104(406961347ns) {ProcessConfigured:135(14246041ns) {StartInitialHandshake:157(14243942ns) {DropHandshakes:469(1973ns) {DropIncomingHandshake:441(444ns) DropOutgoingHandshake:456(378ns)} GenerateSessionVirtualId:988(449ns) PrepareNewSessionHandshake:476(2609ns) {SwitchToState:178(997ns)}}}}}
2025-11-11T17:19:24.802155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518667600019500:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.809795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518667600019507:2059] [node 6] ICP22 created new session: [5:7571518667600019562:2059]
2025-11-11T17:19:24.814014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019566:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:24.817830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019565:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:24.822187Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518667600019565:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.746704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518666626937515:2082] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(184824904ns) {Configure:104(184815173ns) {ProcessConfigured:135(59645ns) {StartInitialHandshake:157(56908ns) {DropHandshakes:469(2644ns) {DropIncomingHandshake:441(739ns) DropOutgoingHandshake:456(381ns)} GenerateSessionVirtualId:988(634ns) PrepareNewSessionHandshake:476(1547ns) {SwitchToState:178(777ns)}}}}}
2025-11-11T17:19:24.786910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518659987298117:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(460864965ns) {Configure:104(460862529ns) {ProcessConfigured:135(17023529ns) {StartInitialHandshake:157(13558485ns) {DropHandshakes:469(1507ns) {DropIncomingHandshake:441(320ns) DropOutgoingHandshake:456(191ns)} GenerateSessionVirtualId:988(1168ns) PrepareNewSessionHandshake:476(3172ns) {SwitchToState:178(1423ns)}}}}}
2025-11-11T17:19:24.787390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518661769494599:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(410825364ns) {Configure:104(410813706ns) {ProcessConfigured:135(29313098ns) {StartInitialHandshake:157(29310208ns) {DropHandshakes:469(1468ns) {DropIncomingHandshake:441(343ns) DropOutgoingHandshake:456(251ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:476(2715ns) {SwitchToState:178(764ns)}}}}}
2025-11-11T17:19:24.770455Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518666628607771:2096] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:24.685753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518663850783037:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(415814952ns) {Configure:104(415811141ns) {ProcessConfigured:135(80701015ns) {StartInitialHandshake:157(80697960ns) {DropHandshakes:469(2247ns) {DropIncomingHandshake:441(406ns) DropOutgoingHandshake:456(433ns)} GenerateSessionVirtualId:988(753ns) PrepareNewSessionHandshake:476(3677ns) {SwitchToState:178(1468ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.789974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518664282265451:2082] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.749281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518666626937517:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-11T17:19:24.809625Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.826021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518663305052184:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.782010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518662331970150:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.685889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518663850783037:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.770741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518662333640424:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.813634Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518666064461950:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.790024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:24.707210Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750359:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.707237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518668145750359:2073] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.716037Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518668145750359:2073] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:24.730813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518668145750400:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(215377093ns) {Configure:104(215371285ns) {ProcessConfigured:135(52102ns) {StartInitialHandshake:157(50323ns) {DropHandshakes:469(1459ns) {DropIncomingHandshake:441(326ns) DropOutgoingHandshake:456(297ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:476(1717ns) {SwitchToState:178(505ns)}}}}}
2025-11-11T17:19:24.748852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518668145750418:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50526
2025-11-11T17:19:24.749064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518668145750421:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50526
2025-11-11T17:19:24.786487Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518668145750395:2105] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:24.786605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518668145750422:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50526
2025-11-11T17:19:24.809943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518663850783038:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.813686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:24.814240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518663850783037:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.814295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:24.814337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518663850783037:2048] [node 1] ICP22 created new session: [2:7571518668145750424:2048]
2025-11-11T17:19:24.814376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518668145750424:2048] [node 1] ICS09 handshake done sender: [2:7571518668145750395:2105] self: [2:7571518668145750416:0] peer: [1:7571518666626937506:0] socket: 276 qp: -1
2025-11-11T17:19:24.814405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518668145750424:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:24.814460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:24.823781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518663850783038:2049] [node 3] ICP22 created new session: [2:7571518668145750423:2049]
2025-11-11T17:19:24.835801Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:24.842809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518668145750423:2049] [node 3] ICS09 handshake done sender: [2:7571518668145750359:2073] self: [2:7571518668145750358:0] peer: [3:7571518666628607799:0] socket: 270 qp: -1
2025-11-11T17:19:24.842844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518668145750423:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:24.869677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:24.883664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750365:2077] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.883708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518668145750365:2077] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.893863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750422:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.830078Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518667600019566:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.790068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518664282265451:2082] [node 5] ICP22 created new session: [6:7571518664282265492:2082]
2025-11-11T17:19:24.793834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:24.770816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:24.818101Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518666064461978:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.807043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518662331970150:2048] [node 2] ICP22 created new session: [1:7571518666626937528:2048]
2025-11-11T17:19:24.821778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937517:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.826357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518666626937510:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(371392597ns) {Configure:104(371386387ns) {ProcessConfigured:135(32662321ns) {StartInitialHandshake:157(32660581ns) {DropHandshakes:469(16093641ns) {DropIncomingHandshake:441(995ns) DropOutgoingHandshake:456(265ns)} GenerateSessionVirtualId:988(1367ns) PrepareNewSessionHandshake:476(1631ns) {SwitchToState:178(103ns)}}}}}
2025-11-11T17:19:24.833071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518666626937528:2048] [node 2] ICS09 handshake done sender: [1:7571518666626937507:2107] self: [1:7571518666626937506:0] peer: [2:7571518668145750416:0] socket: 275 qp: -1
2025-11-11T17:19:24.841701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518666626937528:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:24.843477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518662331970151:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(450341632ns) {Configure:104(450335174ns) {ProcessConfigured:135(27859160ns) {StartInitialHandshake:157(27855930ns) {DropHandshakes:469(2377ns) {DropIncomingHandshake:441(454ns) DropOutgoingHandshake:456(476ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:476(2147ns) {SwitchToState:178(930ns)}}}}}
2025-11-11T17:19:24.843659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:24.854865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937519:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.854896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518666626937519:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.857721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937527:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.857754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518666626937527:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.862188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518662331970150:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(88621805ns) {HandleHandshakeStatus:329(88619465ns) {DropIncomingHandshake:441(1116ns) DropOutgoingHandshake:456(688ns) SwitchToState:178(753ns) ProcessPendingSessionEvents:495(114414ns) {ForwardSessionEventToSession:612(61460ns) ForwardSessionEventToSession:612(4025ns) ForwardSessionEventToSession:612(9429ns) ForwardSessionEventToSession:612(2885ns) ForwardSessionEventToSession:612(5978ns) ForwardSessionEventToSession:612(2681ns) ForwardSessionEventToSession:612(2579ns) ForwardSessionEventToSession:612(8164ns) ForwardSessionEventToSession:612(3639ns) ForwardSessionEventToSession:612(2102ns)}}}
2025-11-11T17:19:24.862262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518662331970150:2048] [node 2] ICP18 handshake (actor [1:7571518666626937517:2113]) is too old
2025-11-11T17:19:24.882288Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518666626937519:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:24.889739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937524:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.889776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518666626937524:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.913477Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518666626937527:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:24.916706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518662331970151:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(19906368ns) {IncomingHandshake:261(19905007ns) {DropIncomingHandshake:441(279ns) PrepareNewSessionHandshake:476(848ns) {SwitchToState:178(453ns)} GenerateSessionVirtualId:988(577ns) IssueIncomingHandshakeReply:193(12894ns)}}
2025-11-11T17:19:24.930263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518666626937513:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.930341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:24.930378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518666626937513:2081] [node 5] ICP22 created new session: [1:7571518666626937537:2081]
2025-11-11T17:19:24.930421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518666626937537:2081] [node 5] ICS09 handshake done sender: [1:7571518666626937519:2114] self: [1:7571518666626937518:0] peer: [5:7571518667600019567:0] socket: 305 qp: -1
2025-11-11T17:19:24.930450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518666626937537:2081] [node 5] ICS10 traffic start
2025-11-11T17:19:24.930482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:24.932987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518666626937524:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.934857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518666626937539:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-11T17:19:24.835535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518667600019562:2059] [node 6] ICS09 handshake done sender: [5:7571518667600019537:2099] self: [5:7571518667600019536:0] peer: [6:7571518664282265486:0] socket: 291 qp: -1
2025-11-11T17:19:24.770855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518662333640424:2049] [node 2] ICP22 created new session: [3:7571518666628607807:2049]
2025-11-11T17:19:24.790127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518664282265492:2082] [node 5] ICS09 handshake done sender: [6:7571518664282265482:2111] self: [6:7571518664282265486:0] peer: [5:7571518667600019536:0] socket: 293 qp: -1
2025-11-11T17:19:24.937971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518668145750398:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(389093410ns) {Configure:104(389084988ns) {ProcessConfigured:135(37097698ns) {StartInitialHandshake:157(37093999ns) {DropHandshakes:469(2535ns) {DropIncomingHandshake:441(836ns) DropOutgoingHandshake:456(481ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:476(12151321ns) {SwitchToState:178(2497ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.826636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518661769494601:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.770897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518666628607807:2049] [node 2] ICS09 handshake done sender: [3:7571518666628607771:2096] self: [3:7571518666628607799:0] peer: [2:7571518668145750358:0] socket: 279 qp: -1
2025-11-11T17:19:24.770927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518666628607807:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:24.770959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:24.778565Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518666628607786:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(136697888ns) {Configure:104(136688083ns) {ProcessConfigured:135(11220630ns) {StartInitialHandshake:157(11218258ns) {DropHandshakes:469(1561ns) {DropIncomingHandshake:441(340ns) DropOutgoingHandshake:456(253ns)} GenerateSessionVirtualId:988(374ns) PrepareNewSessionHandshake:476(2340ns) {SwitchToState:178(649ns)}}}}}
2025-11-11T17:19:24.787552Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607800:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.817047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518666628607811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6255
2025-11-11T17:19:24.817140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518666628607812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6255
2025-11-11T17:19:24.825949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607812:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.826055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518662333640423:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.845753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607811:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.870434Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607803:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.870456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518666628607803:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.893326Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518666628607803:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:24.917284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518666628607793:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.917364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:24.917412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518666628607793:2062] [node 5] ICP22 created new session: [3:7571518666628607818:2062]
2025-11-11T17:19:24.917454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518666628607818:2062] [node 5] ICS09 handshake done sender: [3:7571518666628607803:2116] self: [3:7571518666628607802:0] peer: [5:7571518667600019570:0] socket: 309 qp: -1
2025-11-11T17:19:24.917486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518666628607818:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:24.917518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:24.929665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518666628607816:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6255
2025-11-11T17:19:24.969862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518662333640423:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16611956ns)
2025-11-11T17:19:24.969933Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518666628607812:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:24.970139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518662333640423:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.970189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:24.970226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518662333640423:2048] [node 1] ICP22 created new session: [3:7571518666628607820:2048]
2025-11-11T17:19:24.970272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518666628607820:2048] [node 1] ICS09 handshake done sender: [3:7571518666628607812:2120] self: [3:7571518666628607813:0] peer: [1:7571518666626937526:0] socket: 318 qp: -1
2025-11-11T17:19:24.970301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518666628607820:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:24.970329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:24.846364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019568:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:24.945851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518662331970151:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.790161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518664282265492:2082] [node 5] ICS10 traffic start
2025-11-11T17:19:24.790201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:24.883024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518664282265496:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:24.935200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518664282265497:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:24.997822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518664282265495:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:25.005738Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518664282265497:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:25.005962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518664282265496:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:25.006049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518659987298115:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:25.006167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518659987298117:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.943343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518663850783038:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(133227997ns) {HandleHandshakeStatus:329(133225639ns) {DropIncomingHandshake:441(219ns) DropOutgoingHandshake:456(359ns) SwitchToState:178(382ns) ProcessPendingSessionEvents:495(13416636ns) {ForwardSessionEventToSession:612(13367519ns) ForwardSessionEventToSession:612(4494ns) ForwardSessionEventToSession:612(3130ns) ForwardSessionEventToSession:612(5496ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(8005ns) ForwardSessionEventToSession:612(4626ns) ForwardSessionEventToSession:612(2122ns) ForwardSessionEventToSession:612(4361ns) ForwardSessionEventToSession:612(2215ns)}}}
2025-11-11T17:19:24.834994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518661769494599:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.837739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518666064461922:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(416439878ns) {Configure:104(416436319ns) {ProcessConfigured:135(19889734ns) {StartInitialHandshake:157(19887279ns) {DropHandshakes:469(2245ns) {DropIncomingHandshake:441(393ns) DropOutgoingHandshake:456(709ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:476(2979ns) {SwitchToState:178(315ns)}}}}}
2025-11-11T17:19:24.841840Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518666064461934:2081] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.841867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518666064461934:2081] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:24.843563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518661769494600:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.846790Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518666064461934:2081] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:24.882096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518666064461911:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:24.882159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:24.882211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518666064461911:2063] [node 5] ICP22 created new session: [4:7571518666064461986:2063]
2025-11-11T17:19:24.882252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518666064461986:2063] [node 5] ICS09 handshake done sender: [4:7571518666064461934:2081] self: [4:7571518666064461933:0] peer: [5:7571518667600019564:0] socket: 280 qp: -1
2025-11-11T17:19:24.882286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518666064461986:2063] [node 5] ICS10 traffic start
2025-11-11T17:19:24.882343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:24.906954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518666064461922:2065] [node 6] ICP35 event processing took too much time PendingConnection:278(34871073ns) {IncomingHandshake:261(34869894ns) {DropIncomingHandshake:441(248ns) PrepareNewSessionHandshake:476(22961098ns) {SwitchToState:178(2326ns)} GenerateSessionVirtualId:988(589ns) IssueIncomingHandshakeReply:193(6638ns)}}
2025-11-11T17:19:24.930005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518666064461989:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17737
2025-11-11T17:19:24.962638Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518666064461978:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:24.962771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518666064461990:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17737
2025-11-11T17:19:24.994158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518661769494599:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.957792Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937539:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.846419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019569:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:24.973688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:24.943456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518663850783038:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:25.014007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518666628607809:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.994217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:25.034850Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518664282265495:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.849648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518667600019562:2059] [node 6] ICS10 traffic start
2025-11-11T17:19:24.878467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518663305052186:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.894713Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518667600019514:2078] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:24.894763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:24.910059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019573:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:24.917037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518667600019572:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:24.925649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518663305052186:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(35728979ns) {IncomingHandshake:261(35727772ns) {DropIncomingHandshake:441(1338ns) PrepareNewSessionHandshake:476(2223ns) {SwitchToState:178(1555ns)} GenerateSessionVirtualId:988(466ns) IssueIncomingHandshakeReply:193(23928461ns) {SwitchToState:178(1875ns)}}}
2025-11-11T17:19:24.926153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518667600019507:2059] [node 6] ICP35 event processing took too much time PendingConnection:278(156187668ns) {HandleHandshakeStatus:329(156185400ns) {DropIncomingHandshake:441(18050ns) DropOutgoingHandshake:456(283ns) SwitchToState:178(88ns) ProcessPendingSessionEvents:495(114713ns) {ForwardSessionEventToSession:612(62723ns) ForwardSessionEventToSession:612(5925ns) ForwardSessionEventToSession:612(3503ns) ForwardSessionEventToSession:612(3105ns) ForwardSessionEventToSession:612(5172ns) ForwardSessionEventToSession:612(10166ns) ForwardSessionEventToSession:612(2799ns) ForwardSessionEventToSession:612(5074ns) ForwardSessionEventToSession:612(3022ns) ForwardSessionEventToSession:612(5257ns)}}}
2025-11-11T17:19:24.942083Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:24.942894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518667600019565:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:24.943052Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518667600019566:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:24.945717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518667600019500:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.956388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518663305052186:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518667600019566:2118] held: no
2025-11-11T17:19:24.956567Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518667600019568:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:24.956662Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30119) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:24.961693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:24.965695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518667600019500:2063] [node 4] ICP22 created new session: [5:7571518667600019574:2063]
2025-11-11T17:19:24.982013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518663305052184:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30119) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518667600019565:2117] held: no
2025-11-11T17:19:24.982085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518663305052184:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.982123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:24.982164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518663305052184:2048] [node 1] ICP22 created new session: [5:7571518667600019575:2048]
2025-11-11T17:19:24.982207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518667600019575:2048] [node 1] ICS09 handshake done sender: [5:7571518667600019565:2117] self: [5:7571518667600019567:0] peer: [1:7571518666626937518:0] socket: 312 qp: -1
2025-11-11T17:19:24.982247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518667600019575:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:24.982293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:24.982667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518663305052186:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:24.982708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:24.982768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518663305052186:2050] [node 3] ICP22 created new session: [5:7571518667600019577:2050]
2025-11-11T17:19:24.982807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518667600019577:2050] [node 3] ICS09 handshake done sender: [5:7571518667600019566:2118] self: [5:7571518667600019570:0] peer: [3:7571518666628607802:0] socket: 287 qp: -1
2025-11-11T17:19:24.982828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518667600019577:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:24.982875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:24.983261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518663305052185:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:24.989793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518667600019574:2063] [node 4] ICS09 handshake done sender: [5:7571518667600019514:2078] self: [5:7571518667600019564:0] peer: [4:7571518666064461933:0] socket: 281 qp: -1
2025-11-11T17:19:25.005688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518667600019574:2063] [node 4] ICS10 traffic start
2025-11-11T17:19:25.021944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:25.022368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518667600019500:2063] [node 4] ICP35 event processing took too much time PendingConnection:278(76490447ns) {HandleHandshakeStatus:329(76488028ns) {DropIncomingHandshake:441(993ns) DropOutgoingHandshake:456(5376ns) SwitchToState:178(148ns) ProcessPendingSessionEvents:495(103911ns) {ForwardSessionEventToSession:612(49627ns) ForwardSessionEventToSession:612(5641ns) ForwardSessionEventToSession:612(13091ns) ForwardSessionEventToSession:612(2915ns) ForwardSessionEventToSession:612(5010ns) ForwardSessionEventToSession:612(2880ns) ForwardSessionEventToSession:612(2353ns) ForwardSessionEventToSession:612(5805ns) ForwardSessionEventToSession:612(2287ns) ForwardSessionEventToSession:612(4270ns)}}}
2025-11-11T17:19:25.030378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518671894986877:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44299
2025-11-11T17:19:25.069941Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518667600019568:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:24.994055Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518668145750365:2077] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:24.989733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518662331970151:2049] [node 3] ICP22 created new session: [1:7571518666626937542:2049]
2025-11-11T17:19:25.034994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518664282265424:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:25.035651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518668577232797:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:25.035774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518668577232798:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:25.073770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518668577232799:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:25.089720Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518664282265496:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:25.094569Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:25.094812Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518668577232797:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:25.017055Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518666064461950:2096] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:25.014044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518666628607809:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:25.109417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518659987298117:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518664282265496:2118] held: no
2025-11-11T17:19:25.117831Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18757) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:25.126031Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518664282265497:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:24.994115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518666626937515:2082] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:25.013476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:25.015647Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518666626937531:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:25.015665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518666626937531:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:25.017110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518666626937515:2082] [node 4] ICP22 created new session: [1:7571518670921904839:2082]
2025-11-11T17:19:25.021778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518666626937542:2049] [node 3] ICS09 handshake done sender: [1:7571518666626937527:2116] self: [1:7571518666626937526:0] peer: [3:7571518666628607813:0] socket: 317 qp: -1
2025-11-11T17:19:25.026170Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518666626937531:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:25.029642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518666626937542:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:25.033718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518670921904839:2082] [node 4] ICS09 handshake done sender: [1:7571518666626937524:2115] self: [1:7571518666626937523:0] peer: [4:7571518666064461981:0] socket: 311 qp: -1
2025-11-11T17:19:25.037756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:25.041724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518670921904839:2082] [node 4] ICS10 traffic start
2025-11-11T17:19:25.042151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518666626937510:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:25.053777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:25.062153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518666626937515:2082] [node 4] ICP35 event processing took too much time PendingConnection:278(88302644ns) {HandleHandshakeStatus:329(67858664ns) {DropIncomingHandshake:441(17673ns) DropOutgoingHandshake:456(517ns) SwitchToState:178(453ns) ProcessPendingSessionEvents:495(81082ns) {ForwardSessionEventToSession:612(43958ns) ForwardSessionEventToSession:612(2708ns) ForwardSessionEventToSession:612(4592ns) ForwardSessionEventToSession:612(2333ns) ForwardSessionEventToSession:612(4266ns) ForwardSessionEventToSession:612(1739ns) ForwardSessionEventToSession:612(4496ns) ForwardSessionEventToSession:612(3707ns) ForwardSessionEventToSession:612(4846ns) ForwardSessionEventToSession:612(1921ns)}}}
2025-11-11T17:19:25.069853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:25.077772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518666626937510:2080] [node 6] ICP22 created new session: [1:7571518670921904843:2080]
2025-11-11T17:19:25.093898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518662331970151:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(147862669ns) {HandleHandshakeStatus:329(147860723ns) {DropIncomingHandshake:441(11285ns) DropOutgoingHandshake:456(144ns) SwitchToState:178(123ns) ProcessPendingSessionEvents:495(19943693ns) {ForwardSessionEventToSession:612(19897155ns) ForwardSessionEventToSession:612(4569ns) ForwardSessionEventToSession:612(7069ns) ForwardSessionEventToSession:612(2803ns) ForwardSessionEventToSession:612(4817ns) ForwardSessionEventToSession:612(7289ns) ForwardSessionEventToSession:612(2891ns) ForwardSessionEventToSession:612(1857ns) ForwardSessionEventToSession:612(3765ns) ForwardSessionEventToSession:612(2036ns)}}}
2025-11-11T17:19:25.097661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518670921904843:2080] [node 6] ICS09 handshake done sender: [1:7571518666626937531:2117] self: [1:7571518666626937529:0] peer: [6:7571518668577232794:0] socket: 331 qp: -1
2025-11-11T17:19:25.105672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518670921904843:2080] [node 6] ICS10 traffic start
2025-11-11T17:19:25.129617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:24.997633Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750420:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:25.030649Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518666628607809:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:25.097912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518663305052185:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:25.025953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518661769494599:2048] [node 1] ICP22 created new session: [4:7571518666064461991:2048]
2025-11-11T17:19:24.997668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518668145750420:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:25.046023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518661769494600:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:25.033705Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518668145750420:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:25.042035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518668145750363:2069] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:25.050377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:25.057797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518668145750400:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:25.057862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:25.057904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518668145750400:2067] [node 5] ICP22 created new session: [2:7571518672440717732:2067]
2025-11-11T17:19:25.057941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518672440717732:2067] [node 5] ICS09 handshake done sender: [2:7571518668145750420:2121] self: [2:7571518668145750419:0] peer: [5:7571518667600019579:0] socket: 273 qp: -1
2025-11-11T17:19:25.057972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518672440717732:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:25.057999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:25.069835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518668145750363:2069] [node 4] ICP22 created new session: [2:7571518672440717731:2069]
2025-11-11T17:19:25.094542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518672440717731:2069] [node 4] ICS09 handshake done sender: [2:7571518668145750365:2077] self: [2:7571518668145750364:0] peer: [4:7571518666064461983:0] socket: 271 qp: -1
2025-11-11T17:19:25.099096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518672440717731:2069] [node 4] ICS10 traffic start
2025-11-11T17:19:25.119019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:25.144839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518668145750429:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:25.144865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518668145750429:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:25.150465Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518668145750363:2069] [node 4] ICP35 event processing took too much time PendingConnection:278(108236494ns) {HandleHandshakeStatus:329(108234149ns) {DropIncomingHandshake:441(14128ns) DropOutgoingHandshake:456(305ns) SwitchToState:178(429ns) ProcessPendingSessionEvents:495(8462077ns) {ForwardSessionEventToSession:612(8418001ns) ForwardSessionEventToSession:612(5487ns) ForwardSessionEventToSession:612(2607ns) ForwardSessionEventToSession:612(4756ns) ForwardSessionEventToSession:612(3316ns) ForwardSessionEventToSession:612(4697ns) ForwardSessionEventToSession:612(5573ns) ForwardSessionEventToSession:612(2185ns) ForwardSessionEventToSession:612(4897ns) ForwardSessionEventToSession:612(1578ns)}}}
2025-11-11T17:19:25.150635Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518668145750429:2124] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:25.097992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:25.098041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518663305052185:2049] [node 2] ICP22 created new session: [5:7571518671894986878:2049]
2025-11-11T17:19:25.098079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518671894986878:2049] [node 2] ICS09 handshake done sender: [5:7571518667600019568:2119] self: [5:7571518667600019579:0] peer: [2:7571518668145750419:0] socket: 306 qp: -1
2025-11-11T17:19:25.098107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518671894986878:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:25.098141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:25.126919Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:25.141865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518663305052185:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:25.162251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518666626937510:2080] [node 6] ICP35 event processing took too much time PendingConnection:278(119877245ns) {HandleHandshakeStatus:329(119875502ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(181ns) SwitchToState:178(126ns) ProcessPendingSessionEvents:495(8291488ns) {ForwardSessionEventToSession:612(8239110ns) ForwardSessionEventToSession:612(4733ns) ForwardSessionEventToSession:612(7985ns) ForwardSessionEventToSession:612(3335ns) ForwardSessionEventToSession:612(5044ns) ForwardSessionEventToSession:612(2897ns) ForwardSessionEventToSession:612(2652ns) ForwardSessionEventToSession:612(7543ns) ForwardSessionEventToSession:612(2731ns) ForwardSessionEventToSession:612(2475ns)}}}
2025-11-11T17:19:25.138151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518659987298116:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:25.047528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518666628607786:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:25.153828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518668145750398:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:25.169700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:25.046062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:25.046104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518661769494600:2049] [node 2] ICP22 created new session: [4:7571518670359429288:2049]
2025-11-11T17:19:25.046143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518670359429288:2049] [node 2] ICS09 handshake done sender: [4:7571518666064461950:2096] self: [4:7571518666064461983:0] peer: [2:7571518668145750364:0] socket: 278 qp: -1
2025-11-11T17:19:25.046171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518670359429288:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:25.046196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:25.053687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518666064461991:2048] [node 1] ICS09 handshake done sender: [4:7571518666064461978:2119] self: [4:7571518666064461981:0] peer: [1:7571518666626937523:0] socket: 313 qp: -1
2025-11-11T17:19:25.053723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518666064461991:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:25.089788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518666064461977:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:25.089823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518666064461977:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:25.090749Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518666064461977:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:25.091006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518666064461922:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:25.091055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:25.091094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518666064461922:2065] [node 6] ICP22 created new session: [4:7571518670359429291:2065]
2025-11-11T17:19:25.091138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518670359429291:2065] [node 6] ICS09 handshake done sender: [4:7571518666064461977:2118] self: [4:7571518666064461975:0] peer: [6:7571518668577232796:0] socket: 298 qp: -1
2025-11-11T17:19:25.091164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518670359429291:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:25.091214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:25.095219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:25.134080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518661769494599:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(149025298ns) {HandleHandshakeStatus:329(149022247ns) {DropIncomingHandshake:441(1717ns) DropOutgoingHandshake:456(12892ns) SwitchToState:178(65ns) ProcessPendingSessionEvents:495(99734ns) {ForwardSessionEventToSession:612(53252ns) ForwardSessionEventToSession:612(5321ns) ForwardSessionEventToSession:612(4431ns) ForwardSessionEventToSession:612(2958ns) ForwardSessionEventToSession:612(9024ns) ForwardSessionEventToSession:612(2538ns) ForwardSessionEventToSession:612(3195ns) ForwardSessionEventToSession:612(4687ns) ForwardSessionEventToSession:612(2483ns) ForwardSessionEventToSession:612(4577ns)}}}
2025-11-11T17:19:25.138247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518659987298115:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:25.047577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:25.047613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518666628607786:2064] [node 6] ICP22 created new session: [3:7571518670923575119:2064]
2025-11-11T17:19:25.047653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518670923575119:2064] [node 6] ICS09 handshake done sender: [3:7571518666628607809:2118] self: [3:7571518666628607806:0] peer: [6:7571518668577232795:0] socket: 322 qp: -1
2025-11-11T17:19:25.047688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518670923575119:2064] [node 6] ICS10 traffic start
2025-11-11T17:19:25.047722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:25.138313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:25.138356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518659987298115:2048] [node 1] ICP22 created new session: [6:7571518668577232802:2048]
2025-11-11T17:19:25.138397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518668577232802:2048] [node 1] ICS09 handshake done sender: [6:7571518664282265497:2119] self: [6:7571518668577232794:0] peer: [1:7571518666626937529:0] socket: 332 qp: -1
2025-11-11T17:19:25.177697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518668145750398:2098] [node 6] ICP22 created new session: [2:7571518672440717736:2098]
2025-11-11T17:19:25.177767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518672440717736:2098] [node 6] ICS09 handshake done sender: [2:7571518668145750429:2124] self: [2:7571518668145750428:0] peer: [6:7571518668577232801:0] socket: 272 qp: -1
2025-11-11T17:19:25.177805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518672440717736:2098] [node 6] ICS10 traffic start
2025-11-11T17:19:25.177862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:25.178272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518668145750398:2098] [node 6] ICP35 event processing took too much time PendingConnection:278(24251673ns) {HandleHandshakeStatus:329(24249449ns) {DropIncomingHandshake:441(12021109ns) DropOutgoingHandshake:456(2072ns) SwitchToState:178(349ns) ProcessPendingSessionEvents:495(92616ns) {ForwardSessionEventToSession:612(46052ns) ForwardSessionEventToSession:612(2913ns) ForwardSessionEventToSession:612(4625ns) ForwardSessionEventToSession:612(2606ns) ForwardSessionEventToSession:612(3008ns) ForwardSessionEventToSession:612(8093ns) ForwardSessionEventToSession:612(2921ns) ForwardSessionEventToSession:612(7363ns) ForwardSessionEventToSession:612(5179ns) ForwardSessionEventToSession:612(3182ns)}}}
2025-11-11T17:19:25.138423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518668577232802:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:25.138477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:25.138947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518668577232800:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:25.153830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518664282265424:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18757) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518664282265495:2117] held: no
2025-11-11T17:19:25.173903Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:25.197844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518659987298116:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518668577232797:2120] held: no
2025-11-11T17:19:25.199620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518659987298117:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(32064867ns) {HandleHandshakeStatus:395(32063002ns) {LogHandshakeFail:472(10421ns) {UpdateErrorStateLog:544(3936ns)} DropOutgoingHandshake:456(1640ns) ProcessPendingSessionEvents:495(17457690ns) {DropSessionEvent:512(9646ns) DropSessionEvent:512(5588ns) DropSessionEvent:512(9612384ns) DropSessionEvent:512(5391ns) DropSessionEvent:512(5951ns) DropSessionEvent:512(11082ns) DropSessionEvent:512(10675ns) DropSessionEvent:512(4656ns) DropSessionEvent:512(5517ns) DropSessionEvent:512(4573ns)}}}
2025-11-11T17:19:25.199686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518659987298117:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:25.201713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518664282265424:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(28085875ns) {HandleHandshakeStatus:395(28083638ns) {LogHandshakeFail:472(15307ns) {UpdateErrorStateLog:544(5534ns)} DropOutgoingHandshake:456(2392ns) ProcessPendingSessionEvents:495(12023392ns) {DropSessionEvent:512(6693ns) DropSessionEvent:512(3530ns) DropSessionEvent:512(4687ns) DropSessionEvent:512(3745ns) DropSessionEvent:512(3734ns) DropSessionEvent:512(3839ns) DropSessionEvent:512(4232ns) DropSessionEvent:512(11043ns) DropSessionEvent:512(11225ns) DropSessionEvent:512(8054560ns)}}}
2025-11-11T17:19:25.201822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518664282265495:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:25.202087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518668577232804:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10751
2025-11-11T17:19:25.205773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:25.205775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518664282265424:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:25.205809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:25.206056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518659987298117:2050] [node 3] ICP22 created new session: [6:7571518668577232805:2050]
2025-11-11T17:19:25.209611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518664282265424:2059] [node 4] ICP22 created new session: [6:7571518668577232806:2059]
2025-11-11T17:19:25.209667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518668577232806:2059] [node 4] ICS09 handshake done sender: [6:7571518664282265495:2117] self: [6:7571518668577232796:0] peer: [4:7571518666064461975:0] socket: 319 qp: -1
2025-11-11T17:19:25.209700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518668577232806:2059] [node 4] ICS10 traffic start
2025-11-11T17:19:25.209777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:25.214019Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518668577232797:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:25.214231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518668577232805:2050] [node 3] ICS09 handshake done sender: [6:7571518664282265496:2118] self: [6:7571518668577232795:0] peer: [3:7571518666628607806:0] socket: 323 qp: -1
2025-11-11T17:19:25.221747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518668577232805:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:25.221805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:25.222024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518659987298117:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(22270708ns) {HandleHandshakeStatus:329(22268504ns) {DropIncomingHandshake:441(663ns) DropOutgoingHandshake:456(217ns) SwitchToState:178(442ns) ProcessPendingSessionEvents:495(377ns)}}
2025-11-11T17:19:25.233656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518659987298116:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:25.233732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:25.233783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518659987298116:2049] [node 2] ICP22 created new session: [6:7571518668577232809:2049]
2025-11-11T17:19:25.233848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518668577232809:2049] [node 2] ICS09 handshake done sender: [6:7571518668577232797:2120] self: [6:7571518668577232801:0] peer: [2:7571518668145750428:0] socket: 282 qp: -1
2025-11-11T17:19:25.233890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518668577232809:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:25.233936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:25.234236Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518664282265424:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(24249247ns) {HandleHandshakeStatus:329(24247094ns) {DropIncomingHandshake:441(522ns) DropOutgoingHandshake:456(251ns) SwitchToState:178(303ns) ProcessPendingSessionEvents:495(468ns)}}
2025-11-11T17:19:25.295228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:25.295447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518668145750424:2048] [node 1] ICS01 socket: 276 reason# EndOfStream
2025-11-11T17:19:25.295488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518663850783037:2048] [node 1] ICP30 unregister session Session# [2:7571518668145750424:2048] VirtualId# [2:7571518668145750416:0]
2025-11-11T17:19:25.295518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518668145750424:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.295597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:25.295637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:25.297367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:25.297553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518666628607820:2048] [node 1] ICS01 socket: 318 reason# EndOfStream
2025-11-11T17:19:25.297599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518662333640423:2048] [node 1] ICP30 unregister session Session# [3:7571518666628607820:2048] VirtualId# [3:7571518666628607813:0]
2025-11-11T17:19:25.297625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518666628607820:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.297683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:25.297713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:25.303057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:25.303246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518667600019575:2048] [node 1] ICS01 socket: 312 reason# EndOfStream
2025-11-11T17:19:25.303286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518663305052184:2048] [node 1] ICP30 unregister session Session# [5:7571518667600019575:2048] VirtualId# [5:7571518667600019567:0]
2025-11-11T17:19:25.303313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518667600019575:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.303380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:25.303417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:25.305840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:25.306018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518668577232802:2048] [node 1] ICS01 socket: 332 reason# EndOfStream
2025-11-11T17:19:25.306047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518659987298115:2048] [node 1] ICP30 unregister session Session# [6:7571518668577232802:2048] VirtualId# [6:7571518668577232794:0]
2025-11-11T17:19:25.306071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518668577232802:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.306128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:25.306162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:25.305802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:25.306017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518666064461991:2048] [node 1] ICS01 socket: 313 reason# EndOfStream
2025-11-11T17:19:25.306049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518661769494599:2048] [node 1] ICP30 unregister session Session# [4:7571518666064461991:2048] VirtualId# [4:7571518666064461981:0]
2025-11-11T17:19:25.306073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518666064461991:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.306128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:25.306162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:25.369256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:25.369486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518666628607807:2049] [node 2] ICS01 socket: 279 reason# EndOfStream
2025-11-11T17:19:25.369528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518662333640424:2049] [node 2] ICP30 unregister session Session# [3:7571518666628607807:2049] VirtualId# [3:7571518666628607799:0]
2025-11-11T17:19:25.369558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518666628607807:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.369886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:25.369948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:25.377796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:25.378015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518670359429288:2049] [node 2] ICS01 socket: 278 reason# EndOfStream
2025-11-11T17:19:25.378056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518661769494600:2049] [node 2] ICP30 unregister session Session# [4:7571518670359429288:2049] VirtualId# [4:7571518666064461983:0]
2025-11-11T17:19:25.378083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518670359429288:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.378151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:25.378188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:25.381729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:25.381732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:25.382013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518671894986878:2049] [node 2] ICS01 socket: 306 reason# EndOfStream
2025-11-11T17:19:25.382065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518663305052185:2049] [node 2] ICP30 unregister session Session# [5:7571518671894986878:2049] VirtualId# [5:7571518667600019579:0]
2025-11-11T17:19:25.382103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518671894986878:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.382228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:25.382296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:25.382016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518668577232809:2049] [node 2] ICS01 socket: 282 reason# EndOfStream
2025-11-11T17:19:25.382061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518659987298116:2049] [node 2] ICP30 unregister session Session# [6:7571518668577232809:2049] VirtualId# [6:7571518668577232801:0]
2025-11-11T17:19:25.382103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518668577232809:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.382204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:25.382257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:25.453743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:25.453978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518666064461979:2050] [node 3] ICS01 socket: 285 reason# EndOfStream
2025-11-11T17:19:25.454017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518661769494601:2050] [node 3] ICP30 unregister session Session# [4:7571518666064461979:2050] VirtualId# [4:7571518666064461961:0]
2025-11-11T17:19:25.454044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518666064461979:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.454117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:25.454154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:25.453764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:25.453978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518667600019577:2050] [node 3] ICS01 socket: 287 reason# EndOfStream
2025-11-11T17:19:25.454014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518663305052186:2050] [node 3] ICP30 unregister session Session# [5:7571518667600019577:2050] VirtualId# [5:7571518667600019570:0]
2025-11-11T17:19:25.454044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518667600019577:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.454117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:25.454154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:25.454274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:25.454429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518668577232805:2050] [node 3] ICS01 socket: 323 reason# EndOfStream
2025-11-11T17:19:25.454478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518659987298117:2050] [node 3] ICP30 unregister session Session# [6:7571518668577232805:2050] VirtualId# [6:7571518668577232795:0]
2025-11-11T17:19:25.454506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518668577232805:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.454556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:25.454594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:25.553702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:25.553702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:25.554026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518668577232806:2059] [node 4] ICS01 socket: 319 reason# EndOfStream
2025-11-11T17:19:25.554086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518664282265424:2059] [node 4] ICP30 unregister session Session# [6:7571518668577232806:2059] VirtualId# [6:7571518668577232796:0]
2025-11-11T17:19:25.554148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518668577232806:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.554283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:25.554352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:25.554716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518667600019574:2063] [node 4] ICS07 socket disconnect 281 reason# EndOfStream
2025-11-11T17:19:25.554771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518667600019574:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.554866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:25.554939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518667600019574:2063] [node 4] ICS15 start handshake
2025-11-11T17:19:25.556360Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18757) Couldn't connect to any resolved address
2025-11-11T17:19:25.557134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518667600019500:2063] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18757) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:25.557244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518667600019574:2063] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:25.615071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:25.615310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518664282265492:2082] [node 5] ICS01 socket: 293 reason# EndOfStream
2025-11-11T17:19:25.615345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518664282265451:2082] [node 5] ICP30 unregister session Session# [6:7571518664282265492:2082] VirtualId# [6:7571518664282265486:0]
2025-11-11T17:19:25.615374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518664282265492:2082] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:25.615456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:25.615493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:25.707870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518672144986587:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:25.707918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518672144986588:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:25.730992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518670306097000:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:25.731048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518670306097001:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:25.750768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518669538084078:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:25.750812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518669538084079:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:25.772020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518668599471467:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:25.772086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518668599471468:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:25.772123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518668599471469:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:25.790504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518670893330565:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:25.790543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518670893330566:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:25.790581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518670893330567:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:25.808338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518671825301416:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:25.808421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518671825301417:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:25.808456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518671825301418:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:26.029903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518676120268772:2106] [node 5] ICP01 ready to work
2025-11-11T17:19:26.034516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518676439953945:2107] [node 5] ICP01 ready to work
2025-11-11T17:19:26.034224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518674601064359:2107] [node 5] ICP01 ready to work
2025-11-11T17:19:26.034823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518676439953946:2108] [node 4] ICP01 ready to work
2025-11-11T17:19:26.202927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518676439953945:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(166165239ns) {Configure:104(166160948ns) {ProcessConfigured:135(36733241ns) {StartInitialHandshake:157(36728008ns) {DropHandshakes:469(2976ns) {DropIncomingHandshake:441(488ns) DropOutgoingHandshake:456(530ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:476(6659ns) {SwitchToState:178(3148ns)}}}}}
2025-11-11T17:19:26.199073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518674601064360:2108] [node 6] ICP01 ready to work
2025-11-11T17:19:26.256346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518674601064361:2109] [node 4] ICP01 ready to work
2025-11-11T17:19:26.256606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518675188297927:2108] [node 4] ICP01 ready to work
2025-11-11T17:19:26.256948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518675188297928:2109] [node 6] ICP01 ready to work
2025-11-11T17:19:26.255426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518672894438844:2108] [node 5] ICP01 ready to work
2025-11-11T17:19:26.266569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518669538084078:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(220572876ns) {Configure:104(208862674ns) {ProcessConfigured:135(65048ns) {StartInitialHandshake:157(60432ns) {DropHandshakes:469(1652ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(320ns)} GenerateSessionVirtualId:988(530ns) PrepareNewSessionHandshake:476(1309ns) {SwitchToState:178(746ns)}}}}}
2025-11-11T17:19:26.267041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518673833051439:2107] [node 4] ICP01 ready to work
2025-11-11T17:19:26.267313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518673833051440:2108] [node 5] ICP01 ready to work
2025-11-11T17:19:26.267551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518673833051441:2109] [node 6] ICP01 ready to work
2025-11-11T17:19:26.278187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518676120268772:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(226065941ns) {Configure:104(226061368ns) {ProcessConfigured:135(6306063ns) {StartInitialHandshake:157(6301460ns) {DropHandshakes:469(2266ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(401ns)} GenerateSessionVirtualId:988(919ns) PrepareNewSessionHandshake:476(3883ns) {SwitchToState:178(1563ns)}}}}}
2025-11-11T17:19:26.279250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518676120268787:2108] [node 4] ICP01 ready to work
2025-11-11T17:19:26.278388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518676439953946:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(62764625ns) {Configure:104(62759040ns) {ProcessConfigured:135(9325006ns) {StartInitialHandshake:157(9322961ns) {DropHandshakes:469(2402ns) {DropIncomingHandshake:441(530ns) DropOutgoingHandshake:456(408ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:476(7339ns) {SwitchToState:178(1725ns)}}}}}
2025-11-11T17:19:26.284844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518672894438845:2115] [node 6] ICP01 ready to work
2025-11-11T17:19:26.281605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518676439953954:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.300503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518675188297927:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(39502045ns) {Configure:104(39499159ns) {ProcessConfigured:135(57385ns) {StartInitialHandshake:157(55620ns) {DropHandshakes:469(708ns) {DropIncomingHandshake:441(175ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:476(2019ns) {SwitchToState:178(520ns)}}}}}
2025-11-11T17:19:26.302647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297937:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.304749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518676439953962:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10802
2025-11-11T17:19:26.305015Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518673833051439:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(35854151ns) {Configure:104(35847300ns) {ProcessConfigured:135(4984429ns) {StartInitialHandshake:157(4982124ns) {DropHandshakes:469(3729662ns) {DropIncomingHandshake:441(1246ns) DropOutgoingHandshake:456(701ns)} GenerateSessionVirtualId:988(2169ns) PrepareNewSessionHandshake:476(1993ns) {SwitchToState:178(118ns)}}}}}
2025-11-11T17:19:26.308405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518673833051450:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50256
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.328232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518676439953963:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10802
2025-11-11T17:19:26.325946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518671825301417:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22995005ns) {Configure:104(22984342ns) {ProcessConfigured:135(3986133ns) {StartInitialHandshake:157(3984539ns) {DropHandshakes:469(1597ns) {DropIncomingHandshake:441(302ns) DropOutgoingHandshake:456(374ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:476(3114ns) {SwitchToState:178(480ns)}}}}}
2025-11-11T17:19:26.337501Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518675188297937:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.337139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518668599471469:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30381707ns) {Configure:104(30374218ns) {ProcessConfigured:135(65035ns) {StartInitialHandshake:157(62212ns) {DropHandshakes:469(2361ns) {DropIncomingHandshake:441(519ns) DropOutgoingHandshake:456(497ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:476(1782ns) {SwitchToState:178(446ns)}}}}}
2025-11-11T17:19:26.338276Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953962:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.338806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518670893330565:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.338589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518674601064359:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(56311957ns) {Configure:104(55749526ns) {ProcessConfigured:135(15616623ns) {StartInitialHandshake:157(15611768ns) {DropHandshakes:469(2593ns) {DropIncomingHandshake:441(615ns) DropOutgoingHandshake:456(627ns)} GenerateSessionVirtualId:988(977ns) PrepareNewSessionHandshake:476(4516ns) {SwitchToState:178(2151ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.344256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297944:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.342966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953951:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.343027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518676439953951:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.346245Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518676439953951:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:26.348061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518676439953945:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.350172Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518673833051450:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.348134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:26.348184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518676439953945:2107] [node 5] ICP22 created new session: [1:7571518676439953966:2107]
2025-11-11T17:19:26.348224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518676439953966:2107] [node 5] ICS09 handshake done sender: [1:7571518676439953951:2111] self: [1:7571518676439953950:0] peer: [5:7571518675188297943:0] socket: 277 qp: -1
2025-11-11T17:19:26.348259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518676439953966:2107] [node 5] ICS10 traffic start
2025-11-11T17:19:26.348298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:26.373964Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:26.389839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953963:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.386460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518670306097000:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(47353222ns) {Configure:104(47348835ns) {ProcessConfigured:135(105489ns) {StartInitialHandshake:157(102559ns) {DropHandshakes:469(3227ns) {DropIncomingHandshake:441(760ns) DropOutgoingHandshake:456(551ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:476(2639ns) {SwitchToState:178(846ns)}}}}}
2025-11-11T17:19:26.385743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518668599471467:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39164044ns) {Configure:104(39159600ns) {ProcessConfigured:135(175266ns) {StartInitialHandshake:157(172550ns) {DropHandshakes:469(6015ns) {DropIncomingHandshake:441(894ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(959ns) PrepareNewSessionHandshake:476(2562ns) {SwitchToState:178(792ns)}}}}}
2025-11-11T17:19:26.393848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518675188297937:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:26.405867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518671825301418:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55104405ns) {Configure:104(55097091ns) {ProcessConfigured:135(12191772ns) {StartInitialHandshake:157(12189258ns) {DropHandshakes:469(965ns) {DropIncomingHandshake:441(196ns) DropOutgoingHandshake:456(197ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:476(2287ns) {SwitchToState:178(1243ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.442430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518668599471468:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(56408962ns) {Configure:104(56403424ns) {ProcessConfigured:135(72757ns) {StartInitialHandshake:157(70114ns) {DropHandshakes:469(3091ns) {DropIncomingHandshake:441(732ns) DropOutgoingHandshake:456(626ns)} GenerateSessionVirtualId:988(829ns) PrepareNewSessionHandshake:476(2057ns) {SwitchToState:178(597ns)}}}}}
2025-11-11T17:19:26.430374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518674601064361:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(40722428ns) {Configure:104(40717865ns) {ProcessConfigured:135(97302ns) {StartInitialHandshake:157(93851ns) {DropHandshakes:469(3550ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(513ns)} GenerateSessionVirtualId:988(1098ns) PrepareNewSessionHandshake:476(2947ns) {SwitchToState:178(1133ns)}}}}}
2025-11-11T17:19:26.413886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518670893330565:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518675188297937:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.414004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518670893330565:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.431610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518674601064372:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.477672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518676120268787:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(69539214ns) {Configure:104(67379882ns) {ProcessConfigured:135(64032ns) {StartInitialHandshake:157(60468ns) {DropHandshakes:469(3408ns) {DropIncomingHandshake:441(876ns) DropOutgoingHandshake:456(667ns)} GenerateSessionVirtualId:988(1397ns) PrepareNewSessionHandshake:476(2115ns) {SwitchToState:178(715ns)}}}}}
2025-11-11T17:19:26.478871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268796:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.470139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518673833051455:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50256
2025-11-11T17:19:26.475991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518676439953970:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10802
2025-11-11T17:19:26.414051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.431969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518674601064373:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.512978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518672894438844:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(62534831ns) {Configure:104(62526477ns) {ProcessConfigured:135(63392ns) {StartInitialHandshake:157(61018ns) {DropHandshakes:469(3145ns) {DropIncomingHandshake:441(656ns) DropOutgoingHandshake:456(597ns)} GenerateSessionVirtualId:988(749ns) PrepareNewSessionHandshake:476(1912ns) {SwitchToState:178(801ns)}}}}}
2025-11-11T17:19:26.515687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438856:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
2025-11-11T17:19:26.515916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438857:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.414097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518670893330565:2048] [node 1] ICP22 created new session: [5:7571518675188297945:2048]
2025-11-11T17:19:26.510177Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953970:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.511855Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518673833051455:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.510329Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953948:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.510352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518676439953948:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.511381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518676439953971:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10802
2025-11-11T17:19:26.517286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518669538084078:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.432104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518674601064374:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20065
2025-11-11T17:19:26.414140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518675188297945:2048] [node 1] ICS09 handshake done sender: [5:7571518675188297937:2117] self: [5:7571518675188297943:0] peer: [1:7571518676439953950:0] socket: 279 qp: -1
2025-11-11T17:19:26.414180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518675188297945:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:26.414219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:26.518099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297948:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.518283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297949:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.518776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518675188297948:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.518886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518670893330566:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.520745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297951:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.479556Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064373:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.479634Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064374:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.479744Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064372:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.506204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518670306097000:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.506305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518674601064378:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20065
2025-11-11T17:19:26.516589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518674601064382:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20065
2025-11-11T17:19:26.519282Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064363:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.519309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518674601064363:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.525958Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518674601064363:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.529530Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518672894438857:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.522600Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518676120268796:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.523081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518671825301417:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.525801Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518676439953948:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.535426Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953956:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.534005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518673833051445:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.535462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518676439953956:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.536174Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953971:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.536321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518672144986587:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.536388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:26.536432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518672144986587:2048] [node 2] ICP22 created new session: [1:7571518676439953975:2048]
2025-11-11T17:19:26.536473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518676439953975:2048] [node 2] ICS09 handshake done sender: [1:7571518676439953948:2109] self: [1:7571518676439953947:0] peer: [2:7571518674601064381:0] socket: 276 qp: -1
2025-11-11T17:19:26.536506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518676439953975:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:26.536539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:26.534047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518673833051445:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.538104Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518675188297949:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.529712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438858:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
2025-11-11T17:19:26.539613Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064367:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.539652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518674601064367:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.543085Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518675188297948:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:26.534892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518673833051457:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50256
2025-11-11T17:19:26.532859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518668599471469:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.537219Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518673833051445:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:26.540535Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518672894438856:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.544611Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518672894438858:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.549708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438863:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
2025-11-11T17:19:26.546430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518670893330566:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.537554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518673833051439:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.552599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518676439953974:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10802
2025-11-11T17:19:26.546491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.550485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518674601064359:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.537630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:26.537725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518673833051439:2107] [node 4] ICP22 created new session: [3:7571518673833051459:2107]
2025-11-11T17:19:26.540062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518673833051459:2107] [node 4] ICS09 handshake done sender: [3:7571518673833051445:2112] self: [3:7571518673833051444:0] peer: [4:7571518672894438862:0] socket: 287 qp: -1
2025-11-11T17:19:26.540244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518673833051459:2107] [node 4] ICS10 traffic start
2025-11-11T17:19:26.540285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:26.553883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518673833051462:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50256
2025-11-11T17:19:26.550587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:26.550635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518674601064359:2107] [node 5] ICP22 created new session: [2:7571518674601064384:2107]
2025-11-11T17:19:26.550680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518674601064384:2107] [node 5] ICS09 handshake done sender: [2:7571518674601064363:2110] self: [2:7571518674601064362:0] peer: [5:7571518675188297950:0] socket: 295 qp: -1
2025-11-11T17:19:26.550714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518674601064384:2107] [node 5] ICS10 traffic start
2025-11-11T17:19:26.550748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:26.552197Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064378:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.552281Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:26.546555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518670893330566:2049] [node 2] ICP22 created new session: [5:7571518675188297954:2049]
2025-11-11T17:19:26.546624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518675188297954:2049] [node 2] ICS09 handshake done sender: [5:7571518675188297948:2120] self: [5:7571518675188297950:0] peer: [2:7571518674601064362:0] socket: 298 qp: -1
2025-11-11T17:19:26.546659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518675188297954:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:26.556447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518670306097000:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518674601064372:2115] held: no
2025-11-11T17:19:26.546694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:26.555626Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518676439953956:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:26.554028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518668599471467:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.558332Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518673833051455:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:26.558387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297953:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.565201Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518672894438857:2134] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:26.563120Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518674601064372:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:26.565658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518668599471469:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.565715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:26.565758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518668599471469:2050] [node 3] ICP22 created new session: [4:7571518672894438866:2050]
2025-11-11T17:19:26.579742Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.582859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268801:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.582976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268802:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.583111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268803:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.565944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518672894438866:2050] [node 3] ICS09 handshake done sender: [4:7571518672894438857:2134] self: [4:7571518672894438862:0] peer: [3:7571518673833051444:0] socket: 309 qp: -1
2025-11-11T17:19:26.566203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518672894438866:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:26.566242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:26.584105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518670306097000:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.584219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:26.586877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518669538084078:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.586961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:26.584262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518670306097000:2048] [node 1] ICP22 created new session: [2:7571518674601064390:2048]
2025-11-11T17:19:26.584313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518674601064390:2048] [node 1] ICS09 handshake done sender: [2:7571518674601064372:2115] self: [2:7571518674601064381:0] peer: [1:7571518676439953947:0] socket: 292 qp: -1
2025-11-11T17:19:26.584353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518674601064390:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:26.584398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:26.587024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518669538084078:2048] [node 1] ICP22 created new session: [3:7571518673833051463:2048]
2025-11-11T17:19:26.577666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518672144986588:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.590251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518674601064386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20065
2025-11-11T17:19:26.590498Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518674601064367:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:26.591557Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518676120268796:2124] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:26.592782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518671825301417:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.592870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:26.592921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518671825301417:2049] [node 2] ICP22 created new session: [6:7571518676120268804:2049]
2025-11-11T17:19:26.592964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518676120268804:2049] [node 2] ICS09 handshake done sender: [6:7571518676120268796:2124] self: [6:7571518676120268800:0] peer: [2:7571518674601064366:0] socket: 300 qp: -1
2025-11-11T17:19:26.593003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518676120268804:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:26.593041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:26.597832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518672894438853:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.597868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518672894438853:2131] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.602177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518674601064360:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.602254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:26.582101Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518675188297953:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.587074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518673833051463:2048] [node 1] ICS09 handshake done sender: [3:7571518673833051455:2116] self: [3:7571518673833051456:0] peer: [1:7571518676439953955:0] socket: 283 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.577763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:26.581675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518672144986588:2049] [node 3] ICP22 created new session: [1:7571518676439953980:2049]
2025-11-11T17:19:26.584656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518676439953980:2049] [node 3] ICS09 handshake done sender: [1:7571518676439953956:2113] self: [1:7571518676439953955:0] peer: [3:7571518673833051456:0] socket: 280 qp: -1
2025-11-11T17:19:26.584694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518676439953980:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:26.584735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:26.589978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953953:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.590002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518676439953953:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.607169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518674601064360:2108] [node 6] ICP22 created new session: [2:7571518674601064393:2108]
2025-11-11T17:19:26.610006Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518672894438853:2131] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:26.597133Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518676120268802:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.587104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518673833051463:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:26.587137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:26.587814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518673833051439:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:26.588014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518673833051447:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.588037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518673833051447:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.610473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518673833051465:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50256
2025-11-11T17:19:26.582275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297957:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.607249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518674601064393:2108] [node 6] ICS09 handshake done sender: [2:7571518674601064367:2112] self: [2:7571518674601064366:0] peer: [6:7571518676120268800:0] socket: 297 qp: -1
2025-11-11T17:19:26.597574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518676120268772:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.582498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518670893330567:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.621738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518673833051447:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:26.633650Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4547) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:26.647656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953974:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.645691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518672894438844:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.645814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:26.645865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518672894438844:2108] [node 5] ICP22 created new session: [4:7571518672894438870:2108]
2025-11-11T17:19:26.645912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518672894438870:2108] [node 5] ICS09 handshake done sender: [4:7571518672894438853:2131] self: [4:7571518672894438852:0] peer: [5:7571518675188297959:0] socket: 314 qp: -1
2025-11-11T17:19:26.645945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518672894438870:2108] [node 5] ICS10 traffic start
2025-11-11T17:19:26.645986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:26.607295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518674601064393:2108] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.596444Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518675188297957:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.650268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518676120268772:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4547) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518676120268802:2126] held: no
2025-11-11T17:19:26.596579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518675188297927:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.651950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268808:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.599749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297960:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.634631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518673833051440:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.600437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518675188297939:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.600467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518675188297939:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.634722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:26.601526Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518675188297939:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:26.634774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518673833051440:2108] [node 5] ICP22 created new session: [3:7571518673833051469:2108]
2025-11-11T17:19:26.601811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518675188297928:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.657249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438872:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
2025-11-11T17:19:26.657350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438873:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
2025-11-11T17:19:26.609758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:26.634822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518673833051469:2108] [node 5] ICS09 handshake done sender: [3:7571518673833051447:2113] self: [3:7571518673833051446:0] peer: [5:7571518675188297958:0] socket: 299 qp: -1
2025-11-11T17:19:26.601885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:26.653761Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518676439953953:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:26.634867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518673833051469:2108] [node 5] ICS10 traffic start
2025-11-11T17:19:26.662118Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518672894438856:2133] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:26.601931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518675188297928:2109] [node 6] ICP22 created new session: [5:7571518675188297962:2109]
2025-11-11T17:19:26.601976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518675188297962:2109] [node 6] ICS09 handshake done sender: [5:7571518675188297939:2118] self: [5:7571518675188297938:0] peer: [6:7571518676120268807:0] socket: 293 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.602034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518675188297962:2109] [node 6] ICS10 traffic start
2025-11-11T17:19:26.602069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:26.614572Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064386:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.607986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518675188297964:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49937
2025-11-11T17:19:26.618362Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518675188297957:2124] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:26.618616Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518675188297953:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:26.618800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518675188297927:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.660995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518676439953946:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.634902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:26.635721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518673833051467:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50256
2025-11-11T17:19:26.662301Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518676120268801:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.662434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268809:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.618858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:26.664551Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518676120268802:2126] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.618911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518675188297927:2108] [node 4] ICP22 created new session: [5:7571518675188297965:2108]
2025-11-11T17:19:26.618955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518675188297965:2108] [node 4] ICS09 handshake done sender: [5:7571518675188297957:2124] self: [5:7571518675188297959:0] peer: [4:7571518672894438852:0] socket: 328 qp: -1
2025-11-11T17:19:26.618991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518675188297965:2108] [node 4] ICS10 traffic start
2025-11-11T17:19:26.619057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:26.619484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518670893330567:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.619520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:26.619558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518670893330567:2050] [node 3] ICP22 created new session: [5:7571518675188297967:2050]
2025-11-11T17:19:26.619605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518675188297967:2050] [node 3] ICS09 handshake done sender: [5:7571518675188297953:2123] self: [5:7571518675188297958:0] peer: [3:7571518673833051446:0] socket: 321 qp: -1
2025-11-11T17:19:26.619631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518675188297967:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:26.619656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:26.661071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:26.668419Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518673833051467:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.665117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518676120268772:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.665193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:26.665236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518676120268772:2106] [node 5] ICP22 created new session: [6:7571518676120268812:2106]
2025-11-11T17:19:26.665283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518676120268812:2106] [node 5] ICS09 handshake done sender: [6:7571518676120268802:2126] self: [6:7571518676120268807:0] peer: [5:7571518675188297938:0] socket: 326 qp: -1
2025-11-11T17:19:26.665324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518676120268812:2106] [node 5] ICS10 traffic start
2025-11-11T17:19:26.665359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:26.665934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268811:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.666082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518671825301416:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.661120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518676439953946:2108] [node 4] ICP22 created new session: [1:7571518676439953984:2108]
2025-11-11T17:19:26.661444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518676439953984:2108] [node 4] ICS09 handshake done sender: [1:7571518676439953953:2112] self: [1:7571518676439953952:0] peer: [4:7571518672894438864:0] socket: 278 qp: -1
2025-11-11T17:19:26.661483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518676439953984:2108] [node 4] ICS10 traffic start
2025-11-11T17:19:26.661546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:26.674674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518668599471467:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.674789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:26.674837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518668599471467:2048] [node 1] ICP22 created new session: [4:7571518672894438876:2048]
2025-11-11T17:19:26.674888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518672894438876:2048] [node 1] ICS09 handshake done sender: [4:7571518672894438856:2133] self: [4:7571518672894438864:0] peer: [1:7571518676439953952:0] socket: 308 qp: -1
2025-11-11T17:19:26.674932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518672894438876:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:26.674972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:26.679460Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518676120268809:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.683842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518673833051465:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.683160Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518676439953958:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.685205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518669538084079:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.690232Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064369:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.690284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518674601064369:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.695128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518674601064369:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:26.695463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518671825301418:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.695362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518673833051473:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50256
2025-11-11T17:19:26.696573Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518673833051465:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.683201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518676439953958:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.696993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518670306097001:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.695655Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518676120268811:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.697373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:26.697423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518670306097001:2049] [node 3] ICP22 created new session: [2:7571518674601064397:2049]
2025-11-11T17:19:26.689430Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518676439953958:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:26.697858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518674601064397:2049] [node 3] ICS09 handshake done sender: [2:7571518674601064369:2113] self: [2:7571518674601064368:0] peer: [3:7571518673833051472:0] socket: 317 qp: -1
2025-11-11T17:19:26.697254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438875:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
2025-11-11T17:19:26.698473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518669538084079:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.698259Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518672894438872:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.698536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:26.698383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518668599471468:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.698579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518669538084079:2049] [node 2] ICP22 created new session: [3:7571518673833051474:2049]
2025-11-11T17:19:26.698678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518673833051474:2049] [node 2] ICS09 handshake done sender: [3:7571518673833051465:2119] self: [3:7571518673833051472:0] peer: [2:7571518674601064368:0] socket: 329 qp: -1
2025-11-11T17:19:26.698710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518673833051474:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:26.698745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.699049Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518672894438875:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.699611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268815:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.698985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518676439953954:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.699813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518674601064397:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:26.700214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:26.699049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:26.699100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518676439953954:2110] [node 6] ICP22 created new session: [1:7571518676439953987:2110]
2025-11-11T17:19:26.699139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518676439953987:2110] [node 6] ICS09 handshake done sender: [1:7571518676439953958:2114] self: [1:7571518676439953957:0] peer: [6:7571518676120268814:0] socket: 325 qp: -1
2025-11-11T17:19:26.699170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518676439953987:2110] [node 6] ICS10 traffic start
2025-11-11T17:19:26.699206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:26.701782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518676120268787:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:26.701905Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:26.703162Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518676120268801:2125] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:26.703536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518671825301416:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518676120268801:2125] held: no
2025-11-11T17:19:26.703646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518671825301416:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.703732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:26.703780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518671825301416:2048] [node 1] ICP22 created new session: [6:7571518676120268818:2048]
2025-11-11T17:19:26.703303Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518674601064371:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.703353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518674601064371:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.704594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518676120268818:2048] [node 1] ICS09 handshake done sender: [6:7571518676120268801:2125] self: [6:7571518676120268814:0] peer: [1:7571518676439953957:0] socket: 281 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.704645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518676120268818:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:26.704689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:26.708379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268820:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.709000Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518676120268809:2129] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:26.705854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518673833051449:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.705891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518673833051449:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.707577Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518673833051449:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:26.709809Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518672894438855:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:26.709848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518672894438855:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:26.710496Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518674601064371:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:26.710404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518671825301418:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.710465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:26.710515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518671825301418:2050] [node 3] ICP22 created new session: [6:7571518676120268821:2050]
2025-11-11T17:19:26.710662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518676120268821:2050] [node 3] ICS09 handshake done sender: [6:7571518676120268809:2129] self: [6:7571518676120268816:0] peer: [3:7571518673833051448:0] socket: 291 qp: -1
2025-11-11T17:19:26.710696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518676120268821:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.719603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518672894438880:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16963
2025-11-11T17:19:26.719997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518674601064361:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.710732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:26.717847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518673833051441:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.720250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:26.717950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:26.718003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518673833051441:2109] [node 6] ICP22 created new session: [3:7571518673833051477:2109]
2025-11-11T17:19:26.720015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518673833051477:2109] [node 6] ICS09 handshake done sender: [3:7571518673833051449:2114] self: [3:7571518673833051448:0] peer: [6:7571518676120268816:0] socket: 282 qp: -1
2025-11-11T17:19:26.720345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518673833051477:2109] [node 6] ICS10 traffic start
2025-11-11T17:19:26.723312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:26.720309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518674601064361:2109] [node 4] ICP22 created new session: [2:7571518674601064400:2109]
2025-11-11T17:19:26.723313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518674601064400:2109] [node 4] ICS09 handshake done sender: [2:7571518674601064371:2114] self: [2:7571518674601064370:0] peer: [4:7571518672894438878:0] socket: 335 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:26.724955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518676120268823:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55587
2025-11-11T17:19:26.723187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518672894438855:2132] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:26.723363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518674601064400:2109] [node 4] ICS10 traffic start
2025-11-11T17:19:26.723404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:26.727361Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518676120268811:2130] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:26.727617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518676120268787:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.727687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:26.727767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518676120268787:2108] [node 4] ICP22 created new session: [6:7571518676120268824:2108]
2025-11-11T17:19:26.727825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518676120268824:2108] [node 4] ICS09 handshake done sender: [6:7571518676120268811:2130] self: [6:7571518676120268817:0] peer: [4:7571518672894438854:0] socket: 310 qp: -1
2025-11-11T17:19:26.727867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518676120268824:2108] [node 4] ICS10 traffic start
2025-11-11T17:19:26.727907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:26.725908Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:26.730244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518672894438845:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:26.730365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:26.730414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518672894438845:2115] [node 6] ICP22 created new session: [4:7571518672894438882:2115]
2025-11-11T17:19:26.730469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518672894438882:2115] [node 6] ICS09 handshake done sender: [4:7571518672894438855:2132] self: [4:7571518672894438854:0] peer: [6:7571518676120268817:0] socket: 304 qp: -1
2025-11-11T17:19:26.730510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518672894438882:2115] [node 6] ICS10 traffic start
2025-11-11T17:19:26.730542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:26.731081Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518672894438872:2137] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:26.731998Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518668599471468:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518672894438872:2137] held: no
2025-11-11T17:19:26.732086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518668599471468:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:26.732121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:26.732165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518668599471468:2049] [node 2] ICP22 created new session: [4:7571518672894438884:2049]
2025-11-11T17:19:26.732204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518672894438884:2049] [node 2] ICS09 handshake done sender: [4:7571518672894438872:2137] self: [4:7571518672894438878:0] peer: [2:7571518674601064370:0] socket: 284 qp: -1
2025-11-11T17:19:26.732239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518672894438884:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:26.732272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:26.746690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:26.746932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518674601064390:2048] [node 1] ICS01 socket: 292 reason# EndOfStream
2025-11-11T17:19:26.746973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518670306097000:2048] [node 1] ICP30 unregister session Session# [2:7571518674601064390:2048] VirtualId# [2:7571518674601064381:0]
2025-11-11T17:19:26.747005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518674601064390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.747079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:26.747122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:26.747466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:26.747588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518673833051463:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-11-11T17:19:26.747621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518669538084078:2048] [node 1] ICP30 unregister session Session# [3:7571518673833051463:2048] VirtualId# [3:7571518673833051456:0]
2025-11-11T17:19:26.747645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518673833051463:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.747694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:26.747719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:26.750406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:26.750618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518672894438876:2048] [node 1] ICS01 socket: 308 reason# EndOfStream
2025-11-11T17:19:26.750663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518668599471467:2048] [node 1] ICP30 unregister session Session# [4:7571518672894438876:2048] VirtualId# [4:7571518672894438864:0]
2025-11-11T17:19:26.750687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518672894438876:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.750733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:26.750767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:26.753770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:26.754022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518675188297945:2048] [node 1] ICS01 socket: 279 reason# EndOfStream
2025-11-11T17:19:26.754135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518670893330565:2048] [node 1] ICP30 unregister session Session# [5:7571518675188297945:2048] VirtualId# [5:7571518675188297943:0]
2025-11-11T17:19:26.754171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518675188297945:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.754283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:26.754328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:26.753776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:26.754086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518676120268818:2048] [node 1] ICS01 socket: 281 reason# EndOfStream
2025-11-11T17:19:26.754130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518671825301416:2048] [node 1] ICP30 unregister session Session# [6:7571518676120268818:2048] VirtualId# [6:7571518676120268814:0]
2025-11-11T17:19:26.754149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518676120268818:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.754200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:26.754232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:26.841969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:26.842211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518673833051474:2049] [node 2] ICS01 socket: 329 reason# EndOfStream
2025-11-11T17:19:26.842249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518669538084079:2049] [node 2] ICP30 unregister session Session# [3:7571518673833051474:2049] VirtualId# [3:7571518673833051472:0]
2025-11-11T17:19:26.842279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518673833051474:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.842368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:26.842413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:26.849876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:26.850154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518675188297954:2049] [node 2] ICS01 socket: 298 reason# EndOfStream
2025-11-11T17:19:26.850191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518670893330566:2049] [node 2] ICP30 unregister session Session# [5:7571518675188297954:2049] VirtualId# [5:7571518675188297950:0]
2025-11-11T17:19:26.850223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518675188297954:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.850329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:26.850391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:26.850481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:26.850660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518676120268804:2049] [node 2] ICS01 socket: 300 reason# EndOfStream
2025-11-11T17:19:26.850698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518671825301417:2049] [node 2] ICP30 unregister session Session# [6:7571518676120268804:2049] VirtualId# [6:7571518676120268800:0]
2025-11-11T17:19:26.850719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518676120268804:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.850766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:26.850805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:26.850875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:26.851056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518672894438884:2049] [node 2] ICS01 socket: 284 reason# EndOfStream
2025-11-11T17:19:26.851082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518668599471468:2049] [node 2] ICP30 unregister session Session# [4:7571518672894438884:2049] VirtualId# [4:7571518672894438878:0]
2025-11-11T17:19:26.851103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518672894438884:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.851147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:26.851179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:26.890461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:26.890712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518672894438866:2050] [node 3] ICS01 socket: 309 reason# EndOfStream
2025-11-11T17:19:26.890762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518668599471469:2050] [node 3] ICP30 unregister session Session# [4:7571518672894438866:2050] VirtualId# [4:7571518672894438862:0]
2025-11-11T17:19:26.890798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518672894438866:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.890908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:26.890966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:26.891162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:26.891355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518675188297967:2050] [node 3] ICS01 socket: 321 reason# EndOfStream
2025-11-11T17:19:26.891398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518670893330567:2050] [node 3] ICP30 unregister session Session# [5:7571518675188297967:2050] VirtualId# [5:7571518675188297958:0]
2025-11-11T17:19:26.891423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518675188297967:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.891484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:26.891522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:26.892095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:26.892289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518676120268821:2050] [node 3] ICS01 socket: 291 reason# EndOfStream
2025-11-11T17:19:26.892320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518671825301418:2050] [node 3] ICP30 unregister session Session# [6:7571518676120268821:2050] VirtualId# [6:7571518676120268816:0]
2025-11-11T17:19:26.892348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518676120268821:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.892400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:26.892427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:26.921774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:26.922023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518675188297965:2108] [node 4] ICS01 socket: 328 reason# EndOfStream
2025-11-11T17:19:26.922066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518675188297927:2108] [node 4] ICP30 unregister session Session# [5:7571518675188297965:2108] VirtualId# [5:7571518675188297959:0]
2025-11-11T17:19:26.922096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518675188297965:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.922205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:26.922246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:26.922333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:26.922515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518676120268824:2108] [node 4] ICS01 socket: 310 reason# EndOfStream
2025-11-11T17:19:26.922542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518676120268787:2108] [node 4] ICP30 unregister session Session# [6:7571518676120268824:2108] VirtualId# [6:7571518676120268817:0]
2025-11-11T17:19:26.922570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518676120268824:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.922610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:26.922640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:26.969757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:26.970010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518676120268812:2106] [node 5] ICS01 socket: 326 reason# EndOfStream
2025-11-11T17:19:26.970076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518676120268772:2106] [node 5] ICP30 unregister session Session# [6:7571518676120268812:2106] VirtualId# [6:7571518676120268807:0]
2025-11-11T17:19:26.970115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518676120268812:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:26.970259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:26.970335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:27.125766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518677472195892:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:27.125819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518677472195893:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:27.154378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518680739756152:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:27.154413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518680739756153:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:27.185845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518678082208630:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:27.185886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518678082208631:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:27.222391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518677772872260:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:27.222439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518677772872261:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:27.222469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518677772872262:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:27.245921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518680210766329:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:27.245962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518680210766330:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:27.245989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518680210766331:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:27.534721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518679417624742:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:27.534762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518679417624743:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:27.534789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518679417624744:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:27.599129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518679417624758:2061] [node 5] ICP01 ready to work
2025-11-11T17:19:27.591783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518678082208684:2100] [node 4] ICP01 ready to work
2025-11-11T17:19:27.594374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518680739756204:2099] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.635396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518677472195892:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(37011324ns) {Configure:104(37004754ns) {ProcessConfigured:135(81382ns) {StartInitialHandshake:157(65903ns) {DropHandshakes:469(3436ns) {DropIncomingHandshake:441(870ns) DropOutgoingHandshake:456(824ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:476(2128ns) {SwitchToState:178(530ns)}}}}}
2025-11-11T17:19:27.622818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518680739756214:2102] [node 5] ICP01 ready to work
2025-11-11T17:19:27.626258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518679417624763:2059] [node 4] ICP01 ready to work
2025-11-11T17:19:27.741802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518678082208696:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.747627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518678082208701:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40046
2025-11-11T17:19:27.708847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518680210766351:2066] [node 4] ICP01 ready to work
2025-11-11T17:19:27.645732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518677472195953:2103] [node 6] ICP01 ready to work
2025-11-11T17:19:27.623133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518680739756215:2107] [node 6] ICP01 ready to work
2025-11-11T17:19:27.773013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518677772872262:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(83108463ns) {Configure:104(83102649ns) {ProcessConfigured:135(59651ns) {StartInitialHandshake:157(56062ns) {DropHandshakes:469(2881ns) {DropIncomingHandshake:441(676ns) DropOutgoingHandshake:456(607ns)} GenerateSessionVirtualId:988(912ns) PrepareNewSessionHandshake:476(2058ns) {SwitchToState:178(670ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.822995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518679417624763:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(157129705ns) {Configure:104(157116967ns) {ProcessConfigured:135(58369ns) {StartInitialHandshake:157(56370ns) {DropHandshakes:469(2932ns) {DropIncomingHandshake:441(636ns) DropOutgoingHandshake:456(581ns)} GenerateSessionVirtualId:988(772ns) PrepareNewSessionHandshake:476(1840ns) {SwitchToState:178(666ns)}}}}}
2025-11-11T17:19:27.798193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518678082208707:2106] [node 5] ICP01 ready to work
2025-11-11T17:19:27.705427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518680739756152:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(80613100ns) {Configure:104(80609996ns) {ProcessConfigured:135(70471ns) {StartInitialHandshake:157(67481ns) {DropHandshakes:469(5930ns) {DropIncomingHandshake:441(936ns) DropOutgoingHandshake:456(379ns)} GenerateSessionVirtualId:988(1210ns) PrepareNewSessionHandshake:476(1769ns) {SwitchToState:178(566ns)}}}}}
2025-11-11T17:19:27.702068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518677472195957:2104] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.708998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518680210766346:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.753902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518677472195960:2102] [node 5] ICP01 ready to work
2025-11-11T17:19:27.773436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518677772872325:2088] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.789850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518680210766400:2073] [node 6] ICP01 ready to work
2025-11-11T17:19:27.828271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518679417624783:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:27.718908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518680739756204:2099] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(122679602ns) {Configure:104(122673895ns) {ProcessConfigured:135(63287ns) {StartInitialHandshake:157(61037ns) {DropHandshakes:469(2736ns) {DropIncomingHandshake:441(741ns) DropOutgoingHandshake:456(389ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:476(1532ns) {SwitchToState:178(491ns)}}}}}
2025-11-11T17:19:27.774315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518677772872326:2089] [node 6] ICP01 ready to work
2025-11-11T17:19:27.902154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518677772872326:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(125065436ns) {Configure:104(125062009ns) {ProcessConfigured:135(89758ns) {StartInitialHandshake:157(86967ns) {DropHandshakes:469(3429ns) {DropIncomingHandshake:441(834ns) DropOutgoingHandshake:456(634ns)} GenerateSessionVirtualId:988(1288ns) PrepareNewSessionHandshake:476(1701ns) {SwitchToState:178(686ns)}}}}}
2025-11-11T17:19:27.903522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518677772872334:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
2025-11-11T17:19:27.903578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518677772872335:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
2025-11-11T17:19:27.903706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518677772872336:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
2025-11-11T17:19:27.903756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518677772872337:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
2025-11-11T17:19:27.852324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518680739756229:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27425
2025-11-11T17:19:27.852397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518680739756230:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27425
2025-11-11T17:19:27.896444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518680739756231:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.896918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518677472195970:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17695
2025-11-11T17:19:27.896973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518677472195971:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17695
2025-11-11T17:19:27.897089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518677472195972:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17695
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.941950Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518677772872335:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:27.954865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195971:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:27.924104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518678082208708:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40046
2025-11-11T17:19:27.981684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518678082208701:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:27.981855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518678082208631:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:27.954999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518677472195973:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17695
2025-11-11T17:19:27.973896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518679417624744:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(359736284ns) {Configure:104(359729577ns) {ProcessConfigured:135(7754742ns) {StartInitialHandshake:157(7752135ns) {DropHandshakes:469(2052ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(622ns)} GenerateSessionVirtualId:988(646ns) PrepareNewSessionHandshake:476(4251ns) {SwitchToState:178(1686ns)}}}}}
2025-11-11T17:19:27.966152Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518680739756229:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:27.983067Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518679417624783:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:27.977361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518680210766331:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(261098377ns) {Configure:104(261093334ns) {ProcessConfigured:135(88126640ns) {StartInitialHandshake:157(88123106ns) {DropHandshakes:469(3917ns) {DropIncomingHandshake:441(944ns) DropOutgoingHandshake:456(725ns)} GenerateSessionVirtualId:988(1704ns) PrepareNewSessionHandshake:476(5663ns) {SwitchToState:178(2604ns)}}}}}
2025-11-11T17:19:27.973851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518680739756230:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:27.981732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518680739756152:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:27.985770Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518680739756206:2100] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:27.985796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518680739756206:2100] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:27.991187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518680739756215:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(285374015ns) {Configure:104(285368972ns) {ProcessConfigured:135(16189968ns) {StartInitialHandshake:157(16187414ns) {DropHandshakes:469(2411ns) {DropIncomingHandshake:441(404ns) DropOutgoingHandshake:456(622ns)} GenerateSessionVirtualId:988(598ns) PrepareNewSessionHandshake:476(3306ns) {SwitchToState:178(1221ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:27.994876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518678082208707:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(172958234ns) {Configure:104(172951532ns) {ProcessConfigured:135(70313ns) {StartInitialHandshake:157(65434ns) {DropHandshakes:469(3419ns) {DropIncomingHandshake:441(729ns) DropOutgoingHandshake:456(603ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:476(1429ns) {SwitchToState:178(410ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.018888Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518678082208708:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.030734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518677772872261:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(408705425ns) {Configure:104(408699998ns) {ProcessConfigured:135(20794640ns) {StartInitialHandshake:157(20791676ns) {DropHandshakes:469(3308ns) {DropIncomingHandshake:441(719ns) DropOutgoingHandshake:456(679ns)} GenerateSessionVirtualId:988(955ns) PrepareNewSessionHandshake:476(3509ns) {SwitchToState:178(1534ns)}}}}}
2025-11-11T17:19:28.030883Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518677772872337:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.031266Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518677772872336:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.040481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518677772872262:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.018489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518679417624742:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.013332Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518680210766346:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.013721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195972:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.050063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592118:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.018945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518680739756206:2100] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.058333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518682377176009:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40046
2025-11-11T17:19:28.017666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518677472195893:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(224432956ns) {Configure:104(224428899ns) {ProcessConfigured:135(23758018ns) {StartInitialHandshake:157(23754633ns) {DropHandshakes:469(4329536ns) {DropIncomingHandshake:441(2035ns) DropOutgoingHandshake:456(343ns)} GenerateSessionVirtualId:988(2542ns) PrepareNewSessionHandshake:476(3452ns) {SwitchToState:178(1529ns)}}}}}
2025-11-11T17:19:28.013517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518680210766402:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
2025-11-11T17:19:28.022288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518680210766404:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
2025-11-11T17:19:28.050863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518680210766404:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.061885Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518680210766402:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.074361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518680210766330:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(84031757ns) {Configure:104(84025525ns) {ProcessConfigured:135(63616ns) {StartInitialHandshake:157(61010ns) {DropHandshakes:469(3148ns) {DropIncomingHandshake:441(720ns) DropOutgoingHandshake:456(710ns)} GenerateSessionVirtualId:988(1106ns) PrepareNewSessionHandshake:476(991ns) {SwitchToState:178(350ns)}}}}}
2025-11-11T17:19:28.074989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518680210766351:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.075046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518680210766330:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.065698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518677772872262:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18798009ns) {IncomingHandshake:261(6791514ns) {DropIncomingHandshake:441(709ns) PrepareNewSessionHandshake:476(2516ns) {SwitchToState:178(1083ns)} GenerateSessionVirtualId:988(1150ns) IssueIncomingHandshakeReply:193(61150ns) {SwitchToState:178(705ns)}}}
2025-11-11T17:19:28.031453Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195952:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.031485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518677472195952:2106] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.045976Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195962:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.046011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518677472195962:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.046607Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195970:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.046687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195973:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.066005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518677472195962:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.066093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518677772872334:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.105771Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518678082208686:2101] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.105816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518678082208686:2101] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.106234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518682377176008:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40046
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.133966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518684505733709:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
2025-11-11T17:19:28.146030Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518679417624783:2083] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:28.146409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518679417624742:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.146474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:28.146538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518679417624742:2048] [node 1] ICP22 created new session: [6:7571518683712592119:2048]
2025-11-11T17:19:28.146584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518683712592119:2048] [node 1] ICS09 handshake done sender: [6:7571518679417624783:2083] self: [6:7571518683712592117:0] peer: [1:7571518677472195961:0] socket: 297 qp: -1
2025-11-11T17:19:28.146621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518683712592119:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:28.146656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:28.098226Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518680739756231:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.066297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518677772872260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.117695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195964:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.117686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518677772872330:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.134363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518680739756153:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.174073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518682377176009:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.174263Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518678082208686:2101] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:28.117743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518677472195964:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.130168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518677472195953:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.130210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:28.130260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518677472195953:2103] [node 6] ICP22 created new session: [1:7571518681767163279:2103]
2025-11-11T17:19:28.130308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518681767163279:2103] [node 6] ICS09 handshake done sender: [1:7571518677472195962:2112] self: [1:7571518677472195961:0] peer: [6:7571518683712592117:0] socket: 295 qp: -1
2025-11-11T17:19:28.130346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518681767163279:2103] [node 6] ICS10 traffic start
2025-11-11T17:19:28.130383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:28.131532Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518677472195952:2106] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:28.131860Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518677472195964:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:28.132070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518677472195892:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.132119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:28.132162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518677472195892:2048] [node 2] ICP22 created new session: [1:7571518681767163281:2048]
2025-11-11T17:19:28.132204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518681767163281:2048] [node 2] ICS09 handshake done sender: [1:7571518677472195952:2106] self: [1:7571518677472195951:0] peer: [2:7571518680739756234:0] socket: 294 qp: -1
2025-11-11T17:19:28.132234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518681767163281:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:28.132263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:28.132577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518677472195957:2104] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.132623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:28.132662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518677472195957:2104] [node 4] ICP22 created new session: [1:7571518681767163283:2104]
2025-11-11T17:19:28.132699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518681767163283:2104] [node 4] ICS09 handshake done sender: [1:7571518677472195964:2113] self: [1:7571518677472195963:0] peer: [4:7571518682067839641:0] socket: 296 qp: -1
2025-11-11T17:19:28.132725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518681767163283:2104] [node 4] ICS10 traffic start
2025-11-11T17:19:28.132750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:28.134417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.190452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592122:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.197854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518678082208684:2100] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.197962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:28.198031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518678082208684:2100] [node 4] ICP22 created new session: [3:7571518682377176014:2100]
2025-11-11T17:19:28.198081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518682377176014:2100] [node 4] ICS09 handshake done sender: [3:7571518678082208686:2101] self: [3:7571518678082208685:0] peer: [4:7571518682067839639:0] socket: 290 qp: -1
2025-11-11T17:19:28.198130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518682377176014:2100] [node 4] ICS10 traffic start
2025-11-11T17:19:28.198175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:28.198862Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.202824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592123:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.117736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518677772872330:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.121936Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518677772872330:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:28.134469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518680739756153:2049] [node 3] ICP22 created new session: [2:7571518685034723533:2049]
2025-11-11T17:19:28.208677Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518678082208701:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:28.134513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518685034723533:2049] [node 3] ICS09 handshake done sender: [2:7571518680739756206:2100] self: [2:7571518680739756205:0] peer: [3:7571518678082208709:0] socket: 286 qp: -1
2025-11-11T17:19:28.214990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518678082208631:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518678082208701:2113] held: no
2025-11-11T17:19:28.134546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518685034723533:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:28.215071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518678082208631:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.134578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:28.215118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:28.134730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518680739756217:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.134760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518680739756217:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.122257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518677772872325:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.122316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:28.151402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518677772872325:2088] [node 5] ICP22 created new session: [4:7571518682067839644:2088]
2025-11-11T17:19:28.165865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518682067839644:2088] [node 5] ICS09 handshake done sender: [4:7571518677772872330:2113] self: [4:7571518677772872329:0] peer: [5:7571518684505733707:0] socket: 312 qp: -1
2025-11-11T17:19:28.165924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518682067839644:2088] [node 5] ICS10 traffic start
2025-11-11T17:19:28.165981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:28.166470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518677772872325:2088] [node 5] ICP35 event processing took too much time PendingConnection:278(44037035ns) {HandleHandshakeStatus:329(44033287ns) {DropIncomingHandshake:441(14337ns) DropOutgoingHandshake:456(619ns) SwitchToState:178(729ns) ProcessPendingSessionEvents:495(108925ns) {ForwardSessionEventToSession:612(61132ns) ForwardSessionEventToSession:612(5601ns) ForwardSessionEventToSession:612(3313ns) ForwardSessionEventToSession:612(4878ns) ForwardSessionEventToSession:612(2630ns) ForwardSessionEventToSession:612(2818ns) ForwardSessionEventToSession:612(5652ns) ForwardSessionEventToSession:612(4621ns) ForwardSessionEventToSession:612(2287ns) ForwardSessionEventToSession:612(7222ns)}}}
2025-11-11T17:19:28.166758Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.201884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518677772872260:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518677772872336:2117] held: no
2025-11-11T17:19:28.202535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518682067839646:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
2025-11-11T17:19:28.222173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518682067839647:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
2025-11-11T17:19:28.232248Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28754) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.222667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518683712592122:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.215155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518678082208631:2049] [node 2] ICP22 created new session: [3:7571518682377176016:2049]
2025-11-11T17:19:28.215193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518682377176016:2049] [node 2] ICS09 handshake done sender: [3:7571518678082208701:2113] self: [3:7571518678082208709:0] peer: [2:7571518680739756205:0] socket: 289 qp: -1
2025-11-11T17:19:28.215218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518682377176016:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:28.215250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:28.215689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518682377176012:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40046
2025-11-11T17:19:28.215791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518682377176013:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40046
2025-11-11T17:19:28.243811Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518682377176013:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.253664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518682377176012:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.150289Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.173240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518680739756153:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(38663637ns) {HandleHandshakeStatus:329(38660951ns) {DropIncomingHandshake:441(11089ns) DropOutgoingHandshake:456(206ns) SwitchToState:178(271ns) ProcessPendingSessionEvents:495(38336563ns) {ForwardSessionEventToSession:612(33323926ns) ForwardSessionEventToSession:612(4962596ns) ForwardSessionEventToSession:612(5005ns) ForwardSessionEventToSession:612(2341ns) ForwardSessionEventToSession:612(6089ns) ForwardSessionEventToSession:612(3298ns) ForwardSessionEventToSession:612(2769ns) ForwardSessionEventToSession:612(5466ns) ForwardSessionEventToSession:612(6050ns) ForwardSessionEventToSession:612(5214ns)}}}
2025-11-11T17:19:28.181930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518680739756152:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518680739756230:2115] held: no
2025-11-11T17:19:28.190753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518685034723536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27425
2025-11-11T17:19:28.194169Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518680739756217:2108] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:28.208321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518680739756230:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:28.213898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518680739756152:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.213946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:28.213995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518680739756152:2048] [node 1] ICP22 created new session: [2:7571518685034723540:2048]
2025-11-11T17:19:28.214041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518685034723540:2048] [node 1] ICS09 handshake done sender: [2:7571518680739756230:2115] self: [2:7571518680739756234:0] peer: [1:7571518677472195951:0] socket: 300 qp: -1
2025-11-11T17:19:28.214076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518685034723540:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:28.214109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:28.225770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518680739756214:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.226477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:28.233563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518680739756214:2102] [node 5] ICP22 created new session: [2:7571518685034723542:2102]
2025-11-11T17:19:28.233642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518685034723542:2102] [node 5] ICS09 handshake done sender: [2:7571518680739756217:2108] self: [2:7571518680739756216:0] peer: [5:7571518684505733708:0] socket: 287 qp: -1
2025-11-11T17:19:28.233689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518685034723542:2102] [node 5] ICS10 traffic start
2025-11-11T17:19:28.233745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:28.233674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518680210766351:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28754) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518680210766404:2115] held: no
2025-11-11T17:19:28.256764Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.238373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518679417624743:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.258073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518680739756235:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.269698Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195976:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.269766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518677472195976:2119] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.282235Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518677772872335:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:28.256356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518678082208630:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.282979Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518677472195976:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:28.244260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592124:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.258123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518680739756235:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.288446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518682377176023:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40046
2025-11-11T17:19:28.288990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518682067839649:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
2025-11-11T17:19:28.258251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518683712592124:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.306181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518677472195893:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.317999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:28.270989Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518680739756235:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:28.298020Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518677772872336:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:28.294736Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518678082208703:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.258393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518679417624744:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.329874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518677472195893:2049] [node 3] ICP22 created new session: [1:7571518681767163288:2049]
2025-11-11T17:19:28.294785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518678082208703:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.303512Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518682377176012:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:28.310741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518678082208630:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.310793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:28.310849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518678082208630:2048] [node 1] ICP22 created new session: [3:7571518682377176024:2048]
2025-11-11T17:19:28.310892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518682377176024:2048] [node 1] ICS09 handshake done sender: [3:7571518682377176012:2119] self: [3:7571518682377176022:0] peer: [1:7571518677472195975:0] socket: 288 qp: -1
2025-11-11T17:19:28.310929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518682377176024:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:28.310963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:28.282604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592127:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.303999Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518683712592122:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:28.321103Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.325980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518679417624743:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.334205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518683712592123:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.334331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518679417624763:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.288894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518680739756214:2102] [node 5] ICP35 event processing took too much time PendingConnection:278(62960178ns) {HandleHandshakeStatus:329(62956591ns) {DropIncomingHandshake:441(5759ns) DropOutgoingHandshake:456(999ns) SwitchToState:178(989ns) ProcessPendingSessionEvents:495(30935796ns) {ForwardSessionEventToSession:612(8232223ns) ForwardSessionEventToSession:612(6642ns) ForwardSessionEventToSession:612(4769300ns) ForwardSessionEventToSession:612(3857ns) ForwardSessionEventToSession:612(3884ns) ForwardSessionEventToSession:612(8456429ns) ForwardSessionEventToSession:612(3151ns) ForwardSessionEventToSession:612(9423732ns) ForwardSessionEventToSession:612(6532ns) ForwardSessionEventToSession:612(3056ns)}}}
2025-11-11T17:19:28.312332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518677772872262:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518677772872335:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.349862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518681767163288:2049] [node 3] ICS09 handshake done sender: [1:7571518677472195976:2119] self: [1:7571518677472195975:0] peer: [3:7571518682377176022:0] socket: 313 qp: -1
2025-11-11T17:19:28.349919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518681767163288:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.302315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518680739756215:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.349734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:28.325930Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518680210766404:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:28.374081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:28.381695Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518678082208703:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:28.382211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518678082208696:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.382271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:28.382315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518678082208696:2105] [node 6] ICP22 created new session: [3:7571518682377176027:2105]
2025-11-11T17:19:28.382378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518682377176027:2105] [node 6] ICS09 handshake done sender: [3:7571518678082208703:2114] self: [3:7571518678082208702:0] peer: [6:7571518683712592126:0] socket: 337 qp: -1
2025-11-11T17:19:28.382415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518682377176027:2105] [node 6] ICS10 traffic start
2025-11-11T17:19:28.382471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:28.362036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518684505733711:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
2025-11-11T17:19:28.385726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518680210766351:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.385782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.390009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518684505733710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
2025-11-11T17:19:28.390182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518684505733712:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
2025-11-11T17:19:28.390457Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518680210766402:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.323527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518677772872260:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.349801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518679417624743:2049] [node 2] ICP22 created new session: [6:7571518683712592129:2049]
2025-11-11T17:19:28.302374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:28.397730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518680210766351:2066] [node 4] ICP22 created new session: [5:7571518684505733713:2066]
2025-11-11T17:19:28.410090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518677472195893:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(103700343ns) {HandleHandshakeStatus:329(103697927ns) {DropIncomingHandshake:441(14534ns) DropOutgoingHandshake:456(683ns) SwitchToState:178(1074ns) ProcessPendingSessionEvents:495(117416ns) {ForwardSessionEventToSession:612(66069ns) ForwardSessionEventToSession:612(10673ns) ForwardSessionEventToSession:612(6060ns) ForwardSessionEventToSession:612(5367ns) ForwardSessionEventToSession:612(2548ns) ForwardSessionEventToSession:612(2751ns) ForwardSessionEventToSession:612(5573ns) ForwardSessionEventToSession:612(2406ns) ForwardSessionEventToSession:612(2638ns) ForwardSessionEventToSession:612(5299ns)}}}
2025-11-11T17:19:28.323579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.370176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592130:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.373996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518683712592129:2049] [node 2] ICS09 handshake done sender: [6:7571518683712592122:2117] self: [6:7571518683712592125:0] peer: [2:7571518680739756233:0] socket: 332 qp: -1
2025-11-11T17:19:28.393953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518683712592129:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:28.394005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:28.394398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518679417624743:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(80378473ns) {HandleHandshakeStatus:329(80376571ns) {DropIncomingHandshake:441(2062ns) DropOutgoingHandshake:456(5575ns) SwitchToState:178(322ns) ProcessPendingSessionEvents:495(97536ns) {ForwardSessionEventToSession:612(41258ns) ForwardSessionEventToSession:612(4157ns) ForwardSessionEventToSession:612(2333ns) ForwardSessionEventToSession:612(14362ns) ForwardSessionEventToSession:612(8340ns) ForwardSessionEventToSession:612(6510ns) ForwardSessionEventToSession:612(3256ns) ForwardSessionEventToSession:612(2704ns) ForwardSessionEventToSession:612(4499ns) ForwardSessionEventToSession:612(2293ns)}}}
2025-11-11T17:19:28.394511Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.406097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518679417624743:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:28.406533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518679417624744:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518683712592124:2119] held: no
2025-11-11T17:19:28.406835Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518683712592124:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:28.415980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518679417624744:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.416032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:28.416081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518679417624744:2050] [node 3] ICP22 created new session: [6:7571518683712592133:2050]
2025-11-11T17:19:28.416121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518683712592133:2050] [node 3] ICS09 handshake done sender: [6:7571518683712592124:2119] self: [6:7571518683712592126:0] peer: [3:7571518678082208702:0] socket: 314 qp: -1
2025-11-11T17:19:28.416157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518683712592133:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:28.416191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:28.416571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592132:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.302425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518680739756215:2107] [node 6] ICP22 created new session: [2:7571518685034723546:2107]
2025-11-11T17:19:28.421672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518684505733713:2066] [node 4] ICS09 handshake done sender: [5:7571518680210766404:2115] self: [5:7571518684505733707:0] peer: [4:7571518677772872329:0] socket: 315 qp: -1
2025-11-11T17:19:28.421746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518684505733713:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:28.421791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:28.422060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518680210766351:2066] [node 4] ICP35 event processing took too much time PendingConnection:278(44171996ns) {HandleHandshakeStatus:329(36232813ns) {DropIncomingHandshake:441(1694ns) DropOutgoingHandshake:456(549ns) SwitchToState:178(603ns) ProcessPendingSessionEvents:495(824ns)}}
2025-11-11T17:19:28.323623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518677772872260:2048] [node 1] ICP22 created new session: [4:7571518682067839651:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.302473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518685034723546:2107] [node 6] ICS09 handshake done sender: [2:7571518680739756235:2117] self: [2:7571518680739756233:0] peer: [6:7571518683712592125:0] socket: 311 qp: -1
2025-11-11T17:19:28.422535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518680210766330:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.422618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:28.422663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518680210766330:2049] [node 2] ICP22 created new session: [5:7571518684505733716:2049]
2025-11-11T17:19:28.422707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518684505733716:2049] [node 2] ICS09 handshake done sender: [5:7571518680210766402:2113] self: [5:7571518684505733708:0] peer: [2:7571518680739756216:0] socket: 293 qp: -1
2025-11-11T17:19:28.422739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518684505733716:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:28.422778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:28.423351Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518684505733710:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.423767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518684505733712:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.431741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518680210766329:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.434136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518680210766331:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.323684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518682067839651:2048] [node 1] ICS09 handshake done sender: [4:7571518677772872336:2117] self: [4:7571518682067839641:0] peer: [1:7571518677472195963:0] socket: 308 qp: -1
2025-11-11T17:19:28.323723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518682067839651:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:28.323758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:28.324172Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518682067839649:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.324405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518677772872261:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.442708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592135:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.457645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518678082208711:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.457686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518678082208711:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.302511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518685034723546:2107] [node 6] ICS10 traffic start
2025-11-11T17:19:28.326298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518677772872262:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(31365571ns) {HandleHandshakeStatus:395(27935869ns) {LogHandshakeFail:472(15448ns) {UpdateErrorStateLog:544(4731ns)} DropOutgoingHandshake:456(2624ns) ProcessPendingSessionEvents:495(13722793ns) {DropSessionEvent:512(12855ns) DropSessionEvent:512(8290654ns) DropSessionEvent:512(5686ns) DropSessionEvent:512(4856ns) DropSessionEvent:512(3858ns) DropSessionEvent:512(5233ns) DropSessionEvent:512(6517ns) DropSessionEvent:512(5371796ns) DropSessionEvent:512(4537ns) DropSessionEvent:512(7446ns)}}}
2025-11-11T17:19:28.326390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518677772872262:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.326420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:28.326458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518677772872262:2050] [node 3] ICP22 created new session: [4:7571518682067839654:2050]
2025-11-11T17:19:28.326498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518682067839654:2050] [node 3] ICS09 handshake done sender: [4:7571518677772872335:2116] self: [4:7571518682067839639:0] peer: [3:7571518678082208685:0] socket: 307 qp: -1
2025-11-11T17:19:28.326523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518682067839654:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:28.326556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:28.346655Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518677772872333:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.346682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518677772872333:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.423691Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518677772872333:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:28.439748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518682067839656:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21104
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.473714Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.474897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518684505733720:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.445912Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518677472195966:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.475278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518677772872326:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.470186Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518678082208711:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:28.302558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:28.475336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:28.475378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518677772872326:2089] [node 6] ICP22 created new session: [4:7571518682067839658:2089]
2025-11-11T17:19:28.475420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518682067839658:2089] [node 6] ICS09 handshake done sender: [4:7571518677772872333:2114] self: [4:7571518677772872332:0] peer: [6:7571518683712592128:0] socket: 335 qp: -1
2025-11-11T17:19:28.475454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518682067839658:2089] [node 6] ICS10 traffic start
2025-11-11T17:19:28.475485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:28.475872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518677772872261:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518682067839649:2122] held: no
2025-11-11T17:19:28.491700Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518684505733710:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:28.491982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518680210766329:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.492057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:28.492104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518680210766329:2048] [node 1] ICP22 created new session: [5:7571518684505733722:2048]
2025-11-11T17:19:28.492166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518684505733722:2048] [node 1] ICS09 handshake done sender: [5:7571518684505733710:2119] self: [5:7571518684505733718:0] peer: [1:7571518677472195965:0] socket: 333 qp: -1
2025-11-11T17:19:28.492204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518684505733722:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:28.468246Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518683712592132:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.492243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:28.493853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518684505733721:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22579
2025-11-11T17:19:28.445948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518677472195966:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.509857Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518682067839649:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:28.351617Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518680739756225:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.476386Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518683712592123:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:28.517661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518678082208707:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.528991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:28.351675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518680739756225:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.511479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518677772872261:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.453890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518677472195966:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:28.521748Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518684505733702:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.521809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518684505733702:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:28.530172Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518684505733712:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:28.547558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518678082208707:2106] [node 5] ICP22 created new session: [3:7571518682377176030:2106]
2025-11-11T17:19:28.491139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518677472195960:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.505706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:28.505766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518677472195960:2102] [node 5] ICP22 created new session: [1:7571518681767163291:2102]
2025-11-11T17:19:28.505816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518681767163291:2102] [node 5] ICS09 handshake done sender: [1:7571518677472195966:2114] self: [1:7571518677472195965:0] peer: [5:7571518684505733718:0] socket: 304 qp: -1
2025-11-11T17:19:28.505854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518681767163291:2102] [node 5] ICS10 traffic start
2025-11-11T17:19:28.505916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:28.506323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518677472195960:2102] [node 5] ICP35 event processing took too much time PendingConnection:278(29641701ns) {HandleHandshakeStatus:329(15036733ns) {DropIncomingHandshake:441(20459ns) DropOutgoingHandshake:456(495ns) SwitchToState:178(672ns) ProcessPendingSessionEvents:495(94682ns) {ForwardSessionEventToSession:612(48830ns) ForwardSessionEventToSession:612(4903ns) ForwardSessionEventToSession:612(2425ns) ForwardSessionEventToSession:612(2210ns) ForwardSessionEventToSession:612(8027ns) ForwardSessionEventToSession:612(2327ns) ForwardSessionEventToSession:612(5091ns) ForwardSessionEventToSession:612(2333ns) ForwardSessionEventToSession:612(4541ns) ForwardSessionEventToSession:612(5025ns)}}}
2025-11-11T17:19:28.502263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518679417624758:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:28.381797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518680739756225:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:28.390631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518680739756204:2099] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.390674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:28.390719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518680739756204:2099] [node 4] ICP22 created new session: [2:7571518685034723549:2099]
2025-11-11T17:19:28.390764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518685034723549:2099] [node 4] ICS09 handshake done sender: [2:7571518680739756225:2113] self: [2:7571518680739756224:0] peer: [4:7571518682067839653:0] socket: 299 qp: -1
2025-11-11T17:19:28.390798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518685034723549:2099] [node 4] ICS10 traffic start
2025-11-11T17:19:28.390828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:28.412317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518685034723545:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27425
2025-11-11T17:19:28.423589Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518685034723545:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:28.424007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518680739756204:2099] [node 4] ICP18 handshake (actor [2:7571518685034723545:2119]) is too old
2025-11-11T17:19:28.511529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:28.511575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518677772872261:2049] [node 2] ICP22 created new session: [4:7571518682067839660:2049]
2025-11-11T17:19:28.511616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518682067839660:2049] [node 2] ICS09 handshake done sender: [4:7571518682067839649:2122] self: [4:7571518682067839653:0] peer: [2:7571518680739756224:0] socket: 334 qp: -1
2025-11-11T17:19:28.511650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518682067839660:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:28.511705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:28.570048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518682377176030:2106] [node 5] ICS09 handshake done sender: [3:7571518678082208711:2116] self: [3:7571518678082208710:0] peer: [5:7571518684505733719:0] socket: 338 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:28.510418Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28754) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.512469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518679417624763:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.512542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:28.512583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518679417624763:2059] [node 4] ICP22 created new session: [6:7571518683712592137:2059]
2025-11-11T17:19:28.512629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518683712592137:2059] [node 4] ICS09 handshake done sender: [6:7571518683712592123:2118] self: [6:7571518683712592128:0] peer: [4:7571518677772872332:0] socket: 336 qp: -1
2025-11-11T17:19:28.512662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518683712592137:2059] [node 4] ICS10 traffic start
2025-11-11T17:19:28.512724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:28.512958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518679417624763:2059] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:28.577876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518683712592139:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10106
2025-11-11T17:19:28.546635Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:28.557854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518680210766331:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518684505733712:2121] held: no
2025-11-11T17:19:28.557959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518680210766331:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.558020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:28.558066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518680210766331:2050] [node 3] ICP22 created new session: [5:7571518684505733725:2050]
2025-11-11T17:19:28.558105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518684505733725:2050] [node 3] ICS09 handshake done sender: [5:7571518684505733712:2121] self: [5:7571518684505733719:0] peer: [3:7571518678082208710:0] socket: 339 qp: -1
2025-11-11T17:19:28.558143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518684505733725:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:28.558182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:28.570127Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518684505733702:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:28.578836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518680210766400:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:28.578913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:28.578955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518680210766400:2073] [node 6] ICP22 created new session: [5:7571518684505733727:2073]
2025-11-11T17:19:28.578991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518684505733727:2073] [node 6] ICS09 handshake done sender: [5:7571518684505733702:2116] self: [5:7571518684505733701:0] peer: [6:7571518683712592136:0] socket: 330 qp: -1
2025-11-11T17:19:28.579039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518684505733727:2073] [node 6] ICS10 traffic start
2025-11-11T17:19:28.579060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:28.581441Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518683712592132:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:28.578026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518682377176030:2106] [node 5] ICS10 traffic start
2025-11-11T17:19:28.581727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518679417624758:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:28.586118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:28.586191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518679417624758:2061] [node 5] ICP22 created new session: [6:7571518683712592140:2061]
2025-11-11T17:19:28.586233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518683712592140:2061] [node 5] ICS09 handshake done sender: [6:7571518683712592132:2122] self: [6:7571518683712592136:0] peer: [5:7571518684505733701:0] socket: 344 qp: -1
2025-11-11T17:19:28.586270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518683712592140:2061] [node 5] ICS10 traffic start
2025-11-11T17:19:28.586302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:28.578096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:28.580036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518678082208707:2106] [node 5] ICP35 event processing took too much time PendingConnection:278(62176324ns) {HandleHandshakeStatus:329(62174521ns) {DropIncomingHandshake:441(16570ns) DropOutgoingHandshake:456(6176ns) SwitchToState:178(567ns) ProcessPendingSessionEvents:495(1579551ns) {ForwardSessionEventToSession:612(1535168ns) ForwardSessionEventToSession:612(3544ns) ForwardSessionEventToSession:612(5007ns) ForwardSessionEventToSession:612(2380ns) ForwardSessionEventToSession:612(3545ns) ForwardSessionEventToSession:612(5101ns) ForwardSessionEventToSession:612(3261ns) ForwardSessionEventToSession:612(10142ns) ForwardSessionEventToSession:612(2217ns) ForwardSessionEventToSession:612(2139ns)}}}
2025-11-11T17:19:28.637767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:28.638043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518685034723540:2048] [node 1] ICS01 socket: 300 reason# EndOfStream
2025-11-11T17:19:28.638085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518680739756152:2048] [node 1] ICP30 unregister session Session# [2:7571518685034723540:2048] VirtualId# [2:7571518680739756234:0]
2025-11-11T17:19:28.638122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518685034723540:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.638218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:28.638261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:28.647435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:28.647638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518682067839651:2048] [node 1] ICS01 socket: 308 reason# EndOfStream
2025-11-11T17:19:28.647685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518677772872260:2048] [node 1] ICP30 unregister session Session# [4:7571518682067839651:2048] VirtualId# [4:7571518682067839641:0]
2025-11-11T17:19:28.647716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518682067839651:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.647786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:28.647087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:28.647821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:28.647328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518684505733722:2048] [node 1] ICS01 socket: 333 reason# EndOfStream
2025-11-11T17:19:28.647362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518680210766329:2048] [node 1] ICP30 unregister session Session# [5:7571518684505733722:2048] VirtualId# [5:7571518684505733718:0]
2025-11-11T17:19:28.647396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518684505733722:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.647466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:28.647504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:28.647652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:28.647843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518683712592119:2048] [node 1] ICS01 socket: 297 reason# EndOfStream
2025-11-11T17:19:28.647869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518679417624742:2048] [node 1] ICP30 unregister session Session# [6:7571518683712592119:2048] VirtualId# [6:7571518683712592117:0]
2025-11-11T17:19:28.647889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518683712592119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.647927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:28.647958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:28.649681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:28.649876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518682377176024:2048] [node 1] ICS01 socket: 288 reason# EndOfStream
2025-11-11T17:19:28.649904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518678082208630:2048] [node 1] ICP30 unregister session Session# [3:7571518682377176024:2048] VirtualId# [3:7571518682377176022:0]
2025-11-11T17:19:28.649929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518682377176024:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.650003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:28.650043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:28.723764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:28.724011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518682377176016:2049] [node 2] ICS01 socket: 289 reason# EndOfStream
2025-11-11T17:19:28.724049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518678082208631:2049] [node 2] ICP30 unregister session Session# [3:7571518682377176016:2049] VirtualId# [3:7571518678082208709:0]
2025-11-11T17:19:28.724081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518682377176016:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.724148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:28.724183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:28.737816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:28.738066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518682067839660:2049] [node 2] ICS01 socket: 334 reason# EndOfStream
2025-11-11T17:19:28.738107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518677772872261:2049] [node 2] ICP30 unregister session Session# [4:7571518682067839660:2049] VirtualId# [4:7571518682067839653:0]
2025-11-11T17:19:28.738140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518682067839660:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.738223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:28.738265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:28.738347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:28.738536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518683712592129:2049] [node 2] ICS07 socket disconnect 332 reason# EndOfStream
2025-11-11T17:19:28.738561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518683712592129:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.738605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:28.738638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518683712592129:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:28.739561Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address
2025-11-11T17:19:28.739780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518679417624743:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:28.739823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518683712592129:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:28.739857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518679417624743:2049] [node 2] ICP30 unregister session Session# [6:7571518683712592129:2049] VirtualId# [6:7571518683712592125:0]
2025-11-11T17:19:28.739892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:28.739920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518679417624743:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:28.739934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:28.741715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:28.741939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518684505733716:2049] [node 2] ICS07 socket disconnect 293 reason# EndOfStream
2025-11-11T17:19:28.741973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518684505733716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.742075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:28.742118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518684505733716:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:28.745850Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address
2025-11-11T17:19:28.746115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518680210766330:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:28.746168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518684505733716:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:28.746197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518680210766330:2049] [node 2] ICP30 unregister session Session# [5:7571518684505733716:2049] VirtualId# [5:7571518684505733708:0]
2025-11-11T17:19:28.746234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:28.746269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518680210766330:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:28.746281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:28.753702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518679417624743:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:28.757718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518680210766330:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:28.861803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:28.862026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518683712592133:2050] [node 3] ICS01 socket: 314 reason# EndOfStream
2025-11-11T17:19:28.862074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518679417624744:2050] [node 3] ICP30 unregister session Session# [6:7571518683712592133:2050] VirtualId# [6:7571518683712592126:0]
2025-11-11T17:19:28.862107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518683712592133:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.862193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:28.862241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:28.862336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:28.862515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518684505733725:2050] [node 3] ICS01 socket: 339 reason# EndOfStream
2025-11-11T17:19:28.862546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518680210766331:2050] [node 3] ICP30 unregister session Session# [5:7571518684505733725:2050] VirtualId# [5:7571518684505733719:0]
2025-11-11T17:19:28.862569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518684505733725:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.862632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:28.862675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:28.865685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:28.865967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518682067839654:2050] [node 3] ICS01 socket: 307 reason# EndOfStream
2025-11-11T17:19:28.866015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518677772872262:2050] [node 3] ICP30 unregister session Session# [4:7571518682067839654:2050] VirtualId# [4:7571518682067839639:0]
2025-11-11T17:19:28.866050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518682067839654:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.866144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:28.866200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:28.965776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:28.966023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518683712592137:2059] [node 4] ICS01 socket: 336 reason# EndOfStream
2025-11-11T17:19:28.966063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518679417624763:2059] [node 4] ICP30 unregister session Session# [6:7571518683712592137:2059] VirtualId# [6:7571518683712592128:0]
2025-11-11T17:19:28.966095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518683712592137:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.966190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:28.966228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:28.973804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:28.974068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518684505733713:2066] [node 4] ICS01 socket: 315 reason# EndOfStream
2025-11-11T17:19:28.974105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518680210766351:2066] [node 4] ICP30 unregister session Session# [5:7571518684505733713:2066] VirtualId# [5:7571518684505733707:0]
2025-11-11T17:19:28.974136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518684505733713:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:28.974259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:28.974305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:29.033757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:29.034012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518683712592140:2061] [node 5] ICS01 socket: 344 reason# EndOfStream
2025-11-11T17:19:29.034050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518679417624758:2061] [node 5] ICP30 unregister session Session# [6:7571518683712592140:2061] VirtualId# [6:7571518683712592136:0]
2025-11-11T17:19:29.034085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518683712592140:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:29.034211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:29.034259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:29.249837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518688543128491:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:29.249890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518688543128492:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:29.294661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518688885795485:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:29.294719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518688885795486:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:29.357816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518685719436074:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:29.357862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518685719436075:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:29.401809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518687687257270:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:29.401855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518687687257271:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:29.401885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518687687257272:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:29.461922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518685770644374:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:29.461979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518685770644375:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:29.462012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518685770644376:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:29.549967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518689796156424:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:29.550021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518689796156425:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:29.550048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518689796156426:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:29.839026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518687687257271:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(50041724ns) {Configure:104(50036760ns) {ProcessConfigured:135(73838ns) {StartInitialHandshake:157(71463ns) {DropHandshakes:469(3113ns) {DropIncomingHandshake:441(716ns) DropOutgoingHandshake:456(616ns)} GenerateSessionVirtualId:988(914ns) PrepareNewSessionHandshake:476(1832ns) {SwitchToState:178(749ns)}}}}}
2025-11-11T17:19:29.834132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518688885795526:2084] [node 5] ICP01 ready to work
2025-11-11T17:19:29.809554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518685719436103:2073] [node 6] ICP01 ready to work
2025-11-11T17:19:29.853836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518687687257293:2067] [node 5] ICP01 ready to work
2025-11-11T17:19:29.839414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518688543128544:2096] [node 4] ICP01 ready to work
2025-11-11T17:19:29.854185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518685770644390:2061] [node 6] ICP01 ready to work
2025-11-11T17:19:29.839861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518685719436104:2075] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:29.857779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518688543128546:2094] [node 6] ICP01 ready to work
2025-11-11T17:19:29.877809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518688885795530:2082] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:29.861737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518685719436105:2074] [node 4] ICP01 ready to work
2025-11-11T17:19:29.890407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518685770644395:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:29.881165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518688885795540:2087] [node 6] ICP01 ready to work
2025-11-11T17:19:29.858083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518688543128551:2092] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:29.911523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518685719436074:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(76853858ns) {Configure:104(76847940ns) {ProcessConfigured:135(7205406ns) {StartInitialHandshake:157(7201449ns) {DropHandshakes:469(3027ns) {DropIncomingHandshake:441(772ns) DropOutgoingHandshake:456(819ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:476(3380ns) {SwitchToState:178(1597ns)}}}}}
2025-11-11T17:19:29.893898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518685770644394:2062] [node 4] ICP01 ready to work
2025-11-11T17:19:30.006110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518687687257270:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(216611640ns) {Configure:104(216597480ns) {ProcessConfigured:135(79064ns) {StartInitialHandshake:157(75465ns) {DropHandshakes:469(3724ns) {DropIncomingHandshake:441(803ns) DropOutgoingHandshake:456(735ns)} GenerateSessionVirtualId:988(982ns) PrepareNewSessionHandshake:476(2150ns) {SwitchToState:178(711ns)}}}}}
2025-11-11T17:19:29.895198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518688543128544:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(41196053ns) {Configure:104(41192026ns) {ProcessConfigured:135(54755ns) {StartInitialHandshake:157(51969ns) {DropHandshakes:469(3007ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(432ns)} GenerateSessionVirtualId:988(751ns) PrepareNewSessionHandshake:476(2148ns) {SwitchToState:178(1005ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.111929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518693180762857:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:533
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.077779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518694091123755:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.034800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518685719436103:2073] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(194653419ns) {Configure:104(194648994ns) {ProcessConfigured:135(130355144ns) {StartInitialHandshake:157(130351728ns) {DropHandshakes:469(1323ns) {DropIncomingHandshake:441(356ns) DropOutgoingHandshake:456(245ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:476(4841ns) {SwitchToState:178(1955ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:29.959446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518685770644394:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53311411ns) {Configure:104(53306783ns) {ProcessConfigured:135(73363ns) {StartInitialHandshake:157(68790ns) {DropHandshakes:469(3085ns) {DropIncomingHandshake:441(772ns) DropOutgoingHandshake:456(450ns)} GenerateSessionVirtualId:988(1045ns) PrepareNewSessionHandshake:476(2078ns) {SwitchToState:178(723ns)}}}}}
2025-11-11T17:19:30.112129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518693180762858:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.035297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518685719436109:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55142
2025-11-11T17:19:29.910701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518688543128553:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.012603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611714:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-11T17:19:30.125658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518694091123765:2066] [node 4] ICP01 ready to work
2025-11-11T17:19:30.008646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224604:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:29.938087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518688543128551:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(41318620ns) {Configure:104(41310624ns) {ProcessConfigured:135(31860473ns) {StartInitialHandshake:157(31856806ns) {DropHandshakes:469(1427ns) {DropIncomingHandshake:441(384ns) DropOutgoingHandshake:456(318ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:476(4895ns) {SwitchToState:178(2461ns)}}}}}
2025-11-11T17:19:30.146068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123784:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.112223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518693180762859:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:533
2025-11-11T17:19:30.035362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518685719436110:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55142
2025-11-11T17:19:30.008716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224605:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
2025-11-11T17:19:30.143304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518685770644376:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(161366461ns) {Configure:104(161361391ns) {ProcessConfigured:135(52974ns) {StartInitialHandshake:157(50661ns) {DropHandshakes:469(2285ns) {DropIncomingHandshake:441(476ns) DropOutgoingHandshake:456(467ns)} GenerateSessionVirtualId:988(966ns) PrepareNewSessionHandshake:476(2023ns) {SwitchToState:178(737ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.157889Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518685719436110:2078] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.118517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518692838095868:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-11T17:19:30.169519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518685770644395:2064] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.008951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224606:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
2025-11-11T17:19:30.212730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123797:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.245655Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518694091123784:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.130451Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518693180762859:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.118571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518692838095869:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-11T17:19:30.161774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518688543128553:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.161851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518692838095868:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.169723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518692838095869:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.201819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518688543128555:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.201850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518688543128555:2105] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.205627Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518688543128552:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.205669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518688543128552:2103] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.206638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518692838095873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-11T17:19:30.207270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518688543128552:2103] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:30.209336Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518692838095873:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.209375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518688543128491:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.209407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:30.209441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518688543128491:2048] [node 2] ICP22 created new session: [1:7571518692838095875:2048]
2025-11-11T17:19:30.209474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518692838095875:2048] [node 2] ICS09 handshake done sender: [1:7571518688543128552:2103] self: [1:7571518688543128550:0] peer: [2:7571518693180762863:0] socket: 297 qp: -1
2025-11-11T17:19:30.209498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518692838095875:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:30.209521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:30.213704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518688543128563:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-11T17:19:30.221797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518688543128555:2105] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:30.230275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518688543128491:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(20697860ns) {HandleHandshakeStatus:329(20693494ns) {DropIncomingHandshake:441(366ns) DropOutgoingHandshake:456(452ns) SwitchToState:178(539ns) ProcessPendingSessionEvents:495(20403618ns) {ForwardSessionEventToSession:612(20344512ns) ForwardSessionEventToSession:612(8690ns) ForwardSessionEventToSession:612(9246ns) ForwardSessionEventToSession:612(3055ns) ForwardSessionEventToSession:612(5687ns) ForwardSessionEventToSession:612(2482ns) ForwardSessionEventToSession:612(2054ns) ForwardSessionEventToSession:612(4592ns) ForwardSessionEventToSession:612(2786ns) ForwardSessionEventToSession:612(8911ns)}}}
2025-11-11T17:19:30.230591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518688543128491:2048] [node 2] ICP18 handshake (actor [1:7571518692838095873:2119]) is too old
2025-11-11T17:19:30.258173Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518688543128563:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.258272Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518688543128549:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.258292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518688543128549:2102] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.274583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518688543128544:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.274658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:30.274705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518688543128544:2096] [node 4] ICP22 created new session: [1:7571518692838095881:2096]
2025-11-11T17:19:30.274747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518692838095881:2096] [node 4] ICS09 handshake done sender: [1:7571518688543128555:2105] self: [1:7571518688543128554:0] peer: [4:7571518691982224649:0] socket: 298 qp: -1
2025-11-11T17:19:30.274783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518692838095881:2096] [node 4] ICS10 traffic start
2025-11-11T17:19:30.274817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:30.281984Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518688543128549:2102] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:30.241902Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518685719436109:2077] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.265737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518689796156426:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.121670Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518691982224606:2084] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.169681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518685770644375:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.134141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518691982224641:2094] [node 6] ICP01 ready to work
2025-11-11T17:19:30.162338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518688885795485:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(360663646ns) {Configure:104(360660757ns) {ProcessConfigured:135(65725ns) {StartInitialHandshake:157(57048ns) {DropHandshakes:469(2106ns) {DropIncomingHandshake:441(364ns) DropOutgoingHandshake:456(558ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:476(1090ns) {SwitchToState:178(507ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.242244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518685719436074:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.202555Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:30.234716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518685770644375:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518685770644395:2064] held: no
2025-11-11T17:19:30.262962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518690065611714:2083] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.337746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518685770644390:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(455924544ns) {Configure:104(455914764ns) {ProcessConfigured:135(11779103ns) {StartInitialHandshake:157(11776329ns) {DropHandshakes:469(3246ns) {DropIncomingHandshake:441(824ns) DropOutgoingHandshake:456(632ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:476(3370ns) {SwitchToState:178(1315ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.143974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518687687257272:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(349362767ns) {Configure:104(349357681ns) {ProcessConfigured:135(11940349ns) {StartInitialHandshake:157(11937202ns) {DropHandshakes:469(1532ns) {DropIncomingHandshake:441(298ns) DropOutgoingHandshake:456(185ns)} GenerateSessionVirtualId:988(398ns) PrepareNewSessionHandshake:476(2487ns) {SwitchToState:178(1369ns)}}}}}
2025-11-11T17:19:30.162443Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518693180762857:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.271124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518689796156425:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(133442655ns) {Configure:104(133435944ns) {ProcessConfigured:135(11478541ns) {StartInitialHandshake:157(11475340ns) {DropHandshakes:469(1869ns) {DropIncomingHandshake:441(257ns) DropOutgoingHandshake:456(663ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(6629ns) {SwitchToState:178(1594ns)}}}}}
2025-11-11T17:19:30.357636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611749:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-11T17:19:30.365871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518685770644394:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.367545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611753:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-11T17:19:30.367683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611754:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.242352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518690014403448:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55142
2025-11-11T17:19:30.381783Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518690065611753:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.382228Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518685770644395:2064] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.353819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518688543128492:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.382564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518685770644375:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.144153Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518691982224605:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.242499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518690014403449:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55142
2025-11-11T17:19:30.242679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518685719436075:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.261818Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518690014403449:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.274015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518690014403453:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55142
2025-11-11T17:19:30.281677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518685719436105:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(332305200ns) {Configure:104(332301191ns) {ProcessConfigured:135(11942321ns) {StartInitialHandshake:157(11939001ns) {DropHandshakes:469(3104ns) {DropIncomingHandshake:441(685ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(657ns) PrepareNewSessionHandshake:476(3313ns) {SwitchToState:178(1576ns)}}}}}
2025-11-11T17:19:30.283433Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518690014403448:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.283536Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518690014403426:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.283558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518690014403426:2099] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.284816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518685719436110:2078] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:30.285107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518685719436074:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.285161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:30.285203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518685719436074:2048] [node 1] ICP22 created new session: [3:7571518690014403456:2048]
2025-11-11T17:19:30.285263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518690014403456:2048] [node 1] ICS09 handshake done sender: [3:7571518685719436110:2078] self: [3:7571518690014403450:0] peer: [1:7571518688543128548:0] socket: 300 qp: -1
2025-11-11T17:19:30.285298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518690014403456:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:30.285336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:30.301653Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518690014403426:2099] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:30.322022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518685719436104:2075] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(420149093ns) {Configure:104(420142707ns) {ProcessConfigured:135(20098009ns) {StartInitialHandshake:157(20094551ns) {DropHandshakes:469(3065ns) {DropIncomingHandshake:441(508ns) DropOutgoingHandshake:456(638ns)} GenerateSessionVirtualId:988(586ns) PrepareNewSessionHandshake:476(2227ns) {SwitchToState:178(1264ns)}}}}}
2025-11-11T17:19:30.330164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518690014403462:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55142
2025-11-11T17:19:30.345779Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:30.346645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518685719436103:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.346702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:30.346747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518685719436103:2073] [node 6] ICP22 created new session: [3:7571518690014403463:2073]
2025-11-11T17:19:30.346789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518690014403463:2073] [node 6] ICS09 handshake done sender: [3:7571518690014403426:2099] self: [3:7571518685719436107:0] peer: [6:7571518694091123798:0] socket: 312 qp: -1
2025-11-11T17:19:30.346824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518690014403463:2073] [node 6] ICS10 traffic start
2025-11-11T17:19:30.346857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:30.347235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518685719436075:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518685719436109:2077] held: no
2025-11-11T17:19:30.378394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518690014403466:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55142
2025-11-11T17:19:30.397724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518690014403452:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.162769Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518693180762858:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.163010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518688885795485:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.171206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518693180762865:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:533
2025-11-11T17:19:30.182572Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518688885795534:2092] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.305934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123801:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.353900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:30.319489Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518694091123797:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.319821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518689796156424:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.319894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123802:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.333848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518694091123784:2107] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:30.348071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518689796156426:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.348114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:30.348153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518689796156426:2050] [node 3] ICP22 created new session: [6:7571518694091123804:2050]
2025-11-11T17:19:30.348196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518694091123804:2050] [node 3] ICS09 handshake done sender: [6:7571518694091123784:2107] self: [6:7571518694091123798:0] peer: [3:7571518685719436107:0] socket: 317 qp: -1
2025-11-11T17:19:30.348226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518694091123804:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:30.348251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:30.348761Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:30.357841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518689796156426:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:30.383911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123806:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.417958Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518694091123797:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:30.182598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518688885795534:2092] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.156959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224644:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
2025-11-11T17:19:30.397758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518690014403452:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.385801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518685770644374:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.385893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611756:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-11T17:19:30.393774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:30.393828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518685770644375:2049] [node 2] ICP22 created new session: [5:7571518690065611758:2049]
2025-11-11T17:19:30.414188Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518690065611714:2083] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:30.414375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518685770644394:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.414424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:30.414470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518685770644394:2062] [node 4] ICP22 created new session: [5:7571518690065611759:2062]
2025-11-11T17:19:30.414515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518690065611759:2062] [node 4] ICS09 handshake done sender: [5:7571518690065611714:2083] self: [5:7571518690065611755:0] peer: [4:7571518691982224607:0] socket: 308 qp: -1
2025-11-11T17:19:30.414546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518690065611759:2062] [node 4] ICS10 traffic start
2025-11-11T17:19:30.414583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:30.426083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518690065611758:2049] [node 2] ICS09 handshake done sender: [5:7571518685770644395:2064] self: [5:7571518690065611748:0] peer: [2:7571518688885795533:0] socket: 295 qp: -1
2025-11-11T17:19:30.433825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518690065611758:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:30.353943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518688543128492:2049] [node 3] ICP22 created new session: [1:7571518692838095884:2049]
2025-11-11T17:19:30.185801Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518688885795534:2092] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:30.429682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518694091123802:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.176772Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518691982224604:2082] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.442434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518689796156424:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.442488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:30.442540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518689796156424:2048] [node 1] ICP22 created new session: [6:7571518694091123809:2048]
2025-11-11T17:19:30.442590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518694091123809:2048] [node 1] ICS09 handshake done sender: [6:7571518694091123797:2115] self: [6:7571518694091123803:0] peer: [1:7571518688543128556:0] socket: 332 qp: -1
2025-11-11T17:19:30.442624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518694091123809:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:30.442658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:30.443366Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5086) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:30.462213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518689796156424:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:30.448079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:30.377739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518688543128557:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.398514Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518690014403462:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.186112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518688885795526:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.474710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518694091123765:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.181642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518687687257270:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.399005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518690014403452:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:30.448306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518685770644375:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(65652522ns) {HandleHandshakeStatus:329(65650231ns) {DropIncomingHandshake:441(1158ns) DropOutgoingHandshake:456(2326ns) SwitchToState:178(2616ns) ProcessPendingSessionEvents:495(914ns)}}
2025-11-11T17:19:30.399198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518685719436105:2074] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.448403Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5086) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:30.377778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518688543128557:2106] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.457857Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518690065611754:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.466342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611762:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.190115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518687687257271:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.186193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:30.466404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611763:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-11T17:19:30.473869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518685770644374:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5086) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518690065611753:2118] held: no
2025-11-11T17:19:30.474014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518685770644376:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.483719Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518690065611753:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:30.489997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518685770644374:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.385826Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518688543128557:2106] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:30.399245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.212248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224651:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
2025-11-11T17:19:30.212311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224652:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
2025-11-11T17:19:30.212919Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518691982224644:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.223013Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518691982224604:2082] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:30.223171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518687687257271:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.223210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:30.223250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518687687257271:2049] [node 2] ICP22 created new session: [4:7571518691982224654:2049]
2025-11-11T17:19:30.223297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518691982224654:2049] [node 2] ICS09 handshake done sender: [4:7571518691982224604:2082] self: [4:7571518691982224650:0] peer: [2:7571518688885795535:0] socket: 303 qp: -1
2025-11-11T17:19:30.223330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518691982224654:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:30.223362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:30.231250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518687687257270:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19759232ns) {IncomingHandshake:261(19756387ns) {DropIncomingHandshake:441(1231ns) PrepareNewSessionHandshake:476(2175ns) {SwitchToState:178(1105ns)} GenerateSessionVirtualId:988(497ns) IssueIncomingHandshakeReply:193(16025693ns) {SwitchToState:178(2538ns)}}}
2025-11-11T17:19:30.243011Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518691982224605:2083] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:30.285632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518687687257270:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.301805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:30.301860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518687687257270:2048] [node 1] ICP22 created new session: [4:7571518691982224657:2048]
2025-11-11T17:19:30.301904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518691982224657:2048] [node 1] ICS09 handshake done sender: [4:7571518691982224605:2083] self: [4:7571518691982224649:0] peer: [1:7571518688543128554:0] socket: 304 qp: -1
2025-11-11T17:19:30.301951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518691982224657:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:30.301997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:30.302402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518687687257270:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16584051ns) {HandleHandshakeStatus:329(16581908ns) {DropIncomingHandshake:441(2197ns) DropOutgoingHandshake:456(15035ns) SwitchToState:178(880ns) ProcessPendingSessionEvents:495(89087ns) {ForwardSessionEventToSession:612(51074ns) ForwardSessionEventToSession:612(2490ns) ForwardSessionEventToSession:612(4934ns) ForwardSessionEventToSession:612(2699ns) ForwardSessionEventToSession:612(2894ns) ForwardSessionEventToSession:612(6467ns) ForwardSessionEventToSession:612(2469ns) ForwardSessionEventToSession:612(4539ns) ForwardSessionEventToSession:612(2334ns) ForwardSessionEventToSession:612(2251ns)}}}
2025-11-11T17:19:30.303034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224659:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
2025-11-11T17:19:30.369763Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518691982224659:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.370124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518691982224608:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.370150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518691982224608:2085] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.371120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518687687257272:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.378839Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518691982224608:2085] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:30.427664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518687687257293:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.427711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:30.427755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518687687257293:2067] [node 5] ICP22 created new session: [4:7571518691982224663:2067]
2025-11-11T17:19:30.427810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518691982224663:2067] [node 5] ICS09 handshake done sender: [4:7571518691982224608:2085] self: [4:7571518691982224607:0] peer: [5:7571518690065611755:0] socket: 307 qp: -1
2025-11-11T17:19:30.444359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518691982224663:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:30.444395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:30.450137Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:30.494078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518687687257272:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518691982224659:2121] held: no
2025-11-11T17:19:30.494269Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518691982224646:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.494290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518691982224646:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.495159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518691982224665:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45080
2025-11-11T17:19:30.509850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518687687257293:2067] [node 5] ICP35 event processing took too much time PendingConnection:278(56221695ns) {HandleHandshakeStatus:329(56219873ns) {DropIncomingHandshake:441(13885ns) DropOutgoingHandshake:456(306ns) SwitchToState:178(517ns) ProcessPendingSessionEvents:495(9250806ns) {ForwardSessionEventToSession:612(9198560ns) ForwardSessionEventToSession:612(8915ns) ForwardSessionEventToSession:612(2326ns) ForwardSessionEventToSession:612(7403ns) ForwardSessionEventToSession:612(2535ns) ForwardSessionEventToSession:612(2980ns) ForwardSessionEventToSession:612(7216ns) ForwardSessionEventToSession:612(3188ns) ForwardSessionEventToSession:612(2588ns) ForwardSessionEventToSession:612(5165ns)}}}
2025-11-11T17:19:30.510734Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518691982224659:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:30.506343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123812:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.186243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518688885795526:2084] [node 5] ICP22 created new session: [2:7571518693180762867:2084]
2025-11-11T17:19:30.497933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.543261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123814:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.399296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518685719436105:2074] [node 4] ICP22 created new session: [3:7571518690014403469:2074]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.529809Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518691982224646:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:30.558327Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518694091123812:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.558648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518694091123755:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.559159Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518694091123802:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:30.559510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518694091123765:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.559571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:30.559617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518694091123765:2066] [node 4] ICP22 created new session: [6:7571518694091123816:2066]
2025-11-11T17:19:30.559674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518694091123816:2066] [node 4] ICS09 handshake done sender: [6:7571518694091123802:2118] self: [6:7571518694091123811:0] peer: [4:7571518691982224645:0] socket: 340 qp: -1
2025-11-11T17:19:30.559712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518694091123816:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:30.559746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:30.399332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518690014403469:2074] [node 4] ICS09 handshake done sender: [3:7571518690014403452:2117] self: [3:7571518690014403447:0] peer: [4:7571518691982224662:0] socket: 310 qp: -1
2025-11-11T17:19:30.386064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518692838095884:2049] [node 3] ICS09 handshake done sender: [1:7571518688543128549:2102] self: [1:7571518688543128548:0] peer: [3:7571518690014403450:0] socket: 296 qp: -1
2025-11-11T17:19:30.186303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518693180762867:2084] [node 5] ICS09 handshake done sender: [2:7571518688885795534:2092] self: [2:7571518688885795533:0] peer: [5:7571518690065611748:0] socket: 293 qp: -1
2025-11-11T17:19:30.541707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518687687257272:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.402213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518692838095884:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:30.399361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518690014403469:2074] [node 4] ICS10 traffic start
2025-11-11T17:19:30.186336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518693180762867:2084] [node 5] ICS10 traffic start
2025-11-11T17:19:30.514712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518685770644374:2048] [node 1] ICP22 created new session: [5:7571518690065611766:2048]
2025-11-11T17:19:30.537751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518690065611766:2048] [node 1] ICS09 handshake done sender: [5:7571518690065611753:2118] self: [5:7571518690065611757:0] peer: [1:7571518688543128558:0] socket: 343 qp: -1
2025-11-11T17:19:30.399398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:30.401964Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518685719436109:2077] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:30.426776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518685719436075:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.426824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:30.426867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518685719436075:2049] [node 2] ICP22 created new session: [3:7571518690014403471:2049]
2025-11-11T17:19:30.426921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518690014403471:2049] [node 2] ICS09 handshake done sender: [3:7571518685719436109:2077] self: [3:7571518690014403451:0] peer: [2:7571518688885795531:0] socket: 299 qp: -1
2025-11-11T17:19:30.426954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518690014403471:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:30.426989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:30.495328Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518690014403461:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.495357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518690014403461:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.506680Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518690014403461:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:30.533938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518685719436104:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.534007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:30.534056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518685719436104:2075] [node 5] ICP22 created new session: [3:7571518690014403474:2075]
2025-11-11T17:19:30.534107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518690014403474:2075] [node 5] ICS09 handshake done sender: [3:7571518690014403461:2119] self: [3:7571518690014403460:0] peer: [5:7571518690065611765:0] socket: 311 qp: -1
2025-11-11T17:19:30.534149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518690014403474:2075] [node 5] ICS10 traffic start
2025-11-11T17:19:30.534199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:30.541755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:30.571115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123808:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.537798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518690065611766:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:30.565687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:30.574049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518685770644374:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(83978825ns) {HandleHandshakeStatus:329(83976350ns) {DropIncomingHandshake:441(1139ns) DropOutgoingHandshake:456(510ns) SwitchToState:178(656ns) ProcessPendingSessionEvents:495(908ns)}}
2025-11-11T17:19:30.574446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518690065611752:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.574466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518690065611752:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.575531Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:30.418180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:30.186377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:30.541807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518687687257272:2050] [node 3] ICP22 created new session: [4:7571518691982224667:2050]
2025-11-11T17:19:30.593785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518694091123808:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.593955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518689796156425:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:30.541860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518691982224667:2050] [node 3] ICS09 handshake done sender: [4:7571518691982224659:2121] self: [4:7571518691982224662:0] peer: [3:7571518690014403447:0] socket: 309 qp: -1
2025-11-11T17:19:30.541891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518691982224667:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:30.541924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:30.569896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518691982224641:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.569958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:30.570004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518691982224641:2094] [node 6] ICP22 created new session: [4:7571518691982224669:2094]
2025-11-11T17:19:30.570051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518691982224669:2094] [node 6] ICS09 handshake done sender: [4:7571518691982224646:2118] self: [4:7571518691982224645:0] peer: [6:7571518694091123811:0] socket: 339 qp: -1
2025-11-11T17:19:30.570087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518691982224669:2094] [node 6] ICS10 traffic start
2025-11-11T17:19:30.570122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:30.193812Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518688885795536:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.421656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518688543128546:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.626279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518685770644376:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518690065611754:2119] held: no
2025-11-11T17:19:30.642176Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518690065611762:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.421729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:30.423131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518688543128546:2094] [node 6] ICP22 created new session: [1:7571518692838095887:2094]
2025-11-11T17:19:30.430222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518692838095887:2094] [node 6] ICS09 handshake done sender: [1:7571518688543128557:2106] self: [1:7571518688543128556:0] peer: [6:7571518694091123803:0] socket: 328 qp: -1
2025-11-11T17:19:30.430264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518692838095887:2094] [node 6] ICS10 traffic start
2025-11-11T17:19:30.430320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:30.430686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518688543128546:2094] [node 6] ICP35 event processing took too much time PendingConnection:278(24502023ns) {HandleHandshakeStatus:329(24500244ns) {DropIncomingHandshake:441(19827ns) DropOutgoingHandshake:456(585ns) SwitchToState:178(679ns) ProcessPendingSessionEvents:495(92907ns) {ForwardSessionEventToSession:612(47280ns) ForwardSessionEventToSession:612(6485ns) ForwardSessionEventToSession:612(2923ns) ForwardSessionEventToSession:612(2598ns) ForwardSessionEventToSession:612(9304ns) ForwardSessionEventToSession:612(2534ns) ForwardSessionEventToSession:612(4846ns) ForwardSessionEventToSession:612(2280ns) ForwardSessionEventToSession:612(2341ns) ForwardSessionEventToSession:612(4029ns)}}}
2025-11-11T17:19:30.430810Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518688543128562:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.430832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518688543128562:2107] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.432381Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518688543128562:2107] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:30.432564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518688543128551:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.432626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:30.432674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518688543128551:2092] [node 5] ICP22 created new session: [1:7571518692838095890:2092]
2025-11-11T17:19:30.432726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518692838095890:2092] [node 5] ICS09 handshake done sender: [1:7571518688543128562:2107] self: [1:7571518688543128558:0] peer: [5:7571518690065611757:0] socket: 338 qp: -1
2025-11-11T17:19:30.432751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518692838095890:2092] [node 5] ICS10 traffic start
2025-11-11T17:19:30.432786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:30.450002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518688543128492:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(104128557ns) {HandleHandshakeStatus:329(104125652ns) {DropIncomingHandshake:441(18004ns) DropOutgoingHandshake:456(861ns) SwitchToState:178(864ns) ProcessPendingSessionEvents:495(8055564ns) {ForwardSessionEventToSession:612(7999488ns) ForwardSessionEventToSession:612(15115ns) ForwardSessionEventToSession:612(4956ns) ForwardSessionEventToSession:612(4756ns) ForwardSessionEventToSession:612(3228ns) ForwardSessionEventToSession:612(2501ns) ForwardSessionEventToSession:612(5103ns) ForwardSessionEventToSession:612(2418ns) ForwardSessionEventToSession:612(2929ns) ForwardSessionEventToSession:612(5367ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.658613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518690065611767:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.670247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123819:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.193847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518688885795536:2093] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.670767Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518690065611752:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:30.207111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518693180762870:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:533
2025-11-11T17:19:30.207721Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518693180762857:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:30.207870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518688885795485:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.207918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:30.207956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518688885795485:2048] [node 1] ICP22 created new session: [2:7571518693180762871:2048]
2025-11-11T17:19:30.207998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518693180762871:2048] [node 1] ICS09 handshake done sender: [2:7571518693180762857:2114] self: [2:7571518693180762863:0] peer: [1:7571518688543128550:0] socket: 313 qp: -1
2025-11-11T17:19:30.208025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518693180762871:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:30.208054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:30.218675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518688885795540:2087] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(324772394ns) {Configure:104(324766261ns) {ProcessConfigured:135(54211ns) {StartInitialHandshake:157(49149ns) {DropHandshakes:469(2637ns) {DropIncomingHandshake:441(513ns) DropOutgoingHandshake:456(635ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:476(1939ns) {SwitchToState:178(646ns)}}}}}
2025-11-11T17:19:30.218868Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518688885795536:2093] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:30.222106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518688885795530:2082] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.222148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:30.229655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518688885795530:2082] [node 4] ICP22 created new session: [2:7571518693180762876:2082]
2025-11-11T17:19:30.245762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518693180762876:2082] [node 4] ICS09 handshake done sender: [2:7571518688885795536:2093] self: [2:7571518688885795535:0] peer: [4:7571518691982224650:0] socket: 294 qp: -1
2025-11-11T17:19:30.257648Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518693180762865:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.261733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518693180762876:2082] [node 4] ICS10 traffic start
2025-11-11T17:19:30.261788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:30.325762Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518688885795532:2091] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.325799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518688885795532:2091] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.338923Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518688885795532:2091] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:30.339147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518688885795486:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.339206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:30.339246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518688885795486:2049] [node 3] ICP22 created new session: [2:7571518693180762879:2049]
2025-11-11T17:19:30.339289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518693180762879:2049] [node 3] ICS09 handshake done sender: [2:7571518688885795532:2091] self: [2:7571518688885795531:0] peer: [3:7571518690014403451:0] socket: 292 qp: -1
2025-11-11T17:19:30.339322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518693180762879:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:30.339351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:30.437639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518688885795530:2082] [node 4] ICP35 event processing took too much time PendingConnection:278(175544889ns) {HandleHandshakeStatus:329(175542244ns) {DropIncomingHandshake:441(366ns) DropOutgoingHandshake:456(706ns) SwitchToState:178(596ns) ProcessPendingSessionEvents:495(71864107ns) {ForwardSessionEventToSession:612(27969141ns) ForwardSessionEventToSession:612(15907600ns) ForwardSessionEventToSession:612(4299ns) ForwardSessionEventToSession:612(3088ns) ForwardSessionEventToSession:612(7962242ns) ForwardSessionEventToSession:612(3997ns) ForwardSessionEventToSession:612(3196ns) ForwardSessionEventToSession:612(12001595ns) ForwardSessionEventToSession:612(11049ns) ForwardSessionEventToSession:612(7969649ns)}}}
2025-11-11T17:19:30.501612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518688885795530:2082] [node 4] ICP18 handshake (actor [2:7571518693180762865:2118]) is too old
2025-11-11T17:19:30.501709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [2:7571518688885795530:2082] [node 4] ICP35 event processing took too much time StateWork:293(23844183ns) {IncomingHandshake:261(16033129ns)}
2025-11-11T17:19:30.549673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518693180762882:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:533
2025-11-11T17:19:30.573657Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518693180762882:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.627818Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518693180762875:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:30.627861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518693180762875:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:30.673128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518690065611754:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:30.681851Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518693180762875:2120] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:30.683379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518688885795540:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.683432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:30.683470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518688885795540:2087] [node 6] ICP22 created new session: [2:7571518693180762885:2087]
2025-11-11T17:19:30.683534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518693180762885:2087] [node 6] ICS09 handshake done sender: [2:7571518693180762875:2120] self: [2:7571518693180762874:0] peer: [6:7571518694091123818:0] socket: 346 qp: -1
2025-11-11T17:19:30.683565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518693180762885:2087] [node 6] ICS10 traffic start
2025-11-11T17:19:30.683599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:30.681811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518685770644376:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.681901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:30.681958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518685770644376:2050] [node 3] ICP22 created new session: [5:7571518690065611771:2050]
2025-11-11T17:19:30.682096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518685770644390:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:30.682157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:30.682202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518685770644390:2061] [node 6] ICP22 created new session: [5:7571518690065611772:2061]
2025-11-11T17:19:30.682244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518690065611772:2061] [node 6] ICS09 handshake done sender: [5:7571518690065611752:2117] self: [5:7571518690065611751:0] peer: [6:7571518694091123815:0] socket: 289 qp: -1
2025-11-11T17:19:30.682277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518690065611772:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:30.682314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:30.676799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518694091123812:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:30.677050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518694091123755:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.677100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:30.677142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518694091123755:2063] [node 5] ICP22 created new session: [6:7571518694091123821:2063]
2025-11-11T17:19:30.677189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518694091123821:2063] [node 5] ICS09 handshake done sender: [6:7571518694091123812:2121] self: [6:7571518694091123815:0] peer: [5:7571518690065611751:0] socket: 290 qp: -1
2025-11-11T17:19:30.677226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518694091123821:2063] [node 5] ICS10 traffic start
2025-11-11T17:19:30.677260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:30.706690Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:30.709696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518690065611771:2050] [node 3] ICS09 handshake done sender: [5:7571518690065611754:2119] self: [5:7571518690065611765:0] peer: [3:7571518690014403460:0] socket: 316 qp: -1
2025-11-11T17:19:30.709765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518690065611771:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:30.709822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:30.710072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518685770644376:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(28187842ns) {HandleHandshakeStatus:329(28185198ns) {DropIncomingHandshake:441(1274ns) DropOutgoingHandshake:456(616ns) SwitchToState:178(538ns) ProcessPendingSessionEvents:495(765ns)}}
2025-11-11T17:19:30.723483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518689796156425:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518694091123808:2120] held: no
2025-11-11T17:19:30.723973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518694091123820:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8748
2025-11-11T17:19:30.730088Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518694091123808:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:30.730449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518689796156425:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:30.730493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:30.730544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518689796156425:2049] [node 2] ICP22 created new session: [6:7571518694091123823:2049]
2025-11-11T17:19:30.730584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518694091123823:2049] [node 2] ICS09 handshake done sender: [6:7571518694091123808:2120] self: [6:7571518694091123818:0] peer: [2:7571518693180762874:0] socket: 301 qp: -1
2025-11-11T17:19:30.730619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518694091123823:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:30.730654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:30.781904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:30.782143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518693180762871:2048] [node 1] ICS01 socket: 313 reason# EndOfStream
2025-11-11T17:19:30.782183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518688885795485:2048] [node 1] ICP30 unregister session Session# [2:7571518693180762871:2048] VirtualId# [2:7571518693180762863:0]
2025-11-11T17:19:30.782216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518693180762871:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.782285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:30.782326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:30.785716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:30.785951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518690014403456:2048] [node 1] ICS01 socket: 300 reason# EndOfStream
2025-11-11T17:19:30.785989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518685719436074:2048] [node 1] ICP30 unregister session Session# [3:7571518690014403456:2048] VirtualId# [3:7571518690014403450:0]
2025-11-11T17:19:30.786019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518690014403456:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.786108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:30.786153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:30.797741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:30.797954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518691982224657:2048] [node 1] ICS01 socket: 304 reason# EndOfStream
2025-11-11T17:19:30.797814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:30.797992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518687687257270:2048] [node 1] ICP30 unregister session Session# [4:7571518691982224657:2048] VirtualId# [4:7571518691982224649:0]
2025-11-11T17:19:30.798019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518691982224657:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.798093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:30.798134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:30.797737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:30.797977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518690065611766:2048] [node 1] ICS01 socket: 343 reason# EndOfStream
2025-11-11T17:19:30.798008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518685770644374:2048] [node 1] ICP30 unregister session Session# [5:7571518690065611766:2048] VirtualId# [5:7571518690065611757:0]
2025-11-11T17:19:30.798029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518690065611766:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.798093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:30.798134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:30.801833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518694091123809:2048] [node 1] ICS01 socket: 332 reason# EndOfStream
2025-11-11T17:19:30.801890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518689796156424:2048] [node 1] ICP30 unregister session Session# [6:7571518694091123809:2048] VirtualId# [6:7571518694091123803:0]
2025-11-11T17:19:30.801919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518694091123809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.801997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:30.802037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:30.905788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:30.906032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518690014403471:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-11-11T17:19:30.906075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518685719436075:2049] [node 2] ICP30 unregister session Session# [3:7571518690014403471:2049] VirtualId# [3:7571518690014403451:0]
2025-11-11T17:19:30.906107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518690014403471:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.906205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:30.906270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:30.913733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:30.913730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:30.913961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518694091123823:2049] [node 2] ICS01 socket: 301 reason# EndOfStream
2025-11-11T17:19:30.913961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518691982224654:2049] [node 2] ICS01 socket: 303 reason# EndOfStream
2025-11-11T17:19:30.913998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518689796156425:2049] [node 2] ICP30 unregister session Session# [6:7571518694091123823:2049] VirtualId# [6:7571518694091123818:0]
2025-11-11T17:19:30.914025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518694091123823:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.913996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518687687257271:2049] [node 2] ICP30 unregister session Session# [4:7571518691982224654:2049] VirtualId# [4:7571518691982224650:0]
2025-11-11T17:19:30.914098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:30.914025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518691982224654:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.914135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:30.914098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:30.914138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:30.915212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:30.915422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518690065611758:2049] [node 2] ICS01 socket: 295 reason# EndOfStream
2025-11-11T17:19:30.915455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518685770644375:2049] [node 2] ICP30 unregister session Session# [5:7571518690065611758:2049] VirtualId# [5:7571518690065611748:0]
2025-11-11T17:19:30.915478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518690065611758:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.915544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:30.915576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:30.981779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:30.981845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:30.982048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518694091123804:2050] [node 3] ICS01 socket: 317 reason# EndOfStream
2025-11-11T17:19:30.982088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518689796156426:2050] [node 3] ICP30 unregister session Session# [6:7571518694091123804:2050] VirtualId# [6:7571518694091123798:0]
2025-11-11T17:19:30.982059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518691982224667:2050] [node 3] ICS01 socket: 309 reason# EndOfStream
2025-11-11T17:19:30.982127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518694091123804:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.982226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:30.982103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518687687257272:2050] [node 3] ICP30 unregister session Session# [4:7571518691982224667:2050] VirtualId# [4:7571518691982224662:0]
2025-11-11T17:19:30.982133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518691982224667:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.982268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:30.982216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:30.982272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:30.986713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:30.986956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518690065611771:2050] [node 3] ICS01 socket: 316 reason# EndOfStream
2025-11-11T17:19:30.987007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518685770644376:2050] [node 3] ICP30 unregister session Session# [5:7571518690065611771:2050] VirtualId# [5:7571518690065611765:0]
2025-11-11T17:19:30.987041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518690065611771:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:30.987115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:30.987153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:31.061818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:31.062045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518694091123816:2066] [node 4] ICS01 socket: 340 reason# EndOfStream
2025-11-11T17:19:31.062089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518694091123765:2066] [node 4] ICP30 unregister session Session# [6:7571518694091123816:2066] VirtualId# [6:7571518694091123811:0]
2025-11-11T17:19:31.062126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518694091123816:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:31.062209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:31.062250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:31.069721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:31.069940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518690065611759:2062] [node 4] ICS01 socket: 308 reason# EndOfStream
2025-11-11T17:19:31.069987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518685770644394:2062] [node 4] ICP30 unregister session Session# [5:7571518690065611759:2062] VirtualId# [5:7571518690065611755:0]
2025-11-11T17:19:31.070027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518690065611759:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:31.070149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:31.070195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:31.145762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:31.145981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518694091123821:2063] [node 5] ICS01 socket: 290 reason# EndOfStream
2025-11-11T17:19:31.146022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518694091123755:2063] [node 5] ICP30 unregister session Session# [6:7571518694091123821:2063] VirtualId# [6:7571518694091123815:0]
2025-11-11T17:19:31.146052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518694091123821:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:31.146143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:31.146179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:31.285131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518696337884936:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:31.285173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518696337884937:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:31.361937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518695040408715:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:31.361985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518695040408716:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:31.417968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518696051975436:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:31.418014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518696051975437:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:31.500237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518697761283304:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:31.500280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518697761283305:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:31.500303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518697761283306:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:31.553974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518695533542040:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:31.554022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518695533542041:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:31.554051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518695533542042:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:31.829739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518696051975454:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:31.885999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518694368720768:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:31.886045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518694368720769:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:31.886637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518694368720782:2059] [node 5] ICP01 ready to work
2025-11-11T17:19:31.886900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518694368720783:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:31.815478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518696337884970:2074] [node 4] ICP01 ready to work
2025-11-11T17:19:31.869747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518696337884989:2075] [node 6] ICP01 ready to work
2025-11-11T17:19:31.869655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518695040408755:2072] [node 5] ICP01 ready to work
2025-11-11T17:19:31.891232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518695533542060:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:31.974402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518695533542066:2068] [node 6] ICP01 ready to work
2025-11-11T17:19:31.870034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518695040408761:2067] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:31.905883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518697761283323:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:31.905652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518694368720767:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:32.050054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518695533542060:2063] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:31.877858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518696337884993:2076] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.072123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688134:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.025663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518696051975467:2067] [node 5] ICP01 ready to work
2025-11-11T17:19:31.961834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518697761283328:2059] [node 6] ICP01 ready to work
2025-11-11T17:19:31.870465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518695040408762:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.130036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518699828509408:2084] [node 4] ICP01 ready to work
2025-11-11T17:19:32.093694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518700346942779:2073] [node 6] ICP01 ready to work
2025-11-11T17:19:32.133989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518700346942794:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-11T17:19:32.134165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518700346942795:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-11T17:19:32.134258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518700346942796:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-11T17:19:32.061960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518697761283330:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.120540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518696337884936:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(302387652ns) {Configure:104(302383878ns) {ProcessConfigured:135(74757ns) {StartInitialHandshake:157(71291ns) {DropHandshakes:469(6027ns) {DropIncomingHandshake:441(823ns) DropOutgoingHandshake:456(598ns)} GenerateSessionVirtualId:988(623ns) PrepareNewSessionHandshake:476(2736ns) {SwitchToState:178(1164ns)}}}}}
2025-11-11T17:19:31.897753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518695040408760:2069] [node 6] ICP01 ready to work
2025-11-11T17:19:31.981710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518695040408762:2092] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.171827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518695040408715:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(353929103ns) {Configure:104(353919686ns) {ProcessConfigured:135(60910ns) {StartInitialHandshake:157(58352ns) {DropHandshakes:469(3585ns) {DropIncomingHandshake:441(834ns) DropOutgoingHandshake:456(962ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:476(2851ns) {SwitchToState:178(624ns)}}}}}
2025-11-11T17:19:32.173828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518699335376082:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-11T17:19:32.174977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518695040408755:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(277086665ns) {Configure:104(277080957ns) {ProcessConfigured:135(45792ns) {StartInitialHandshake:157(44061ns) {DropHandshakes:469(1282ns) {DropIncomingHandshake:441(283ns) DropOutgoingHandshake:456(153ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:476(587ns) {SwitchToState:178(182ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.192940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509413:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.169667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518698663688134:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.122315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518696337884937:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(252374400ns) {Configure:104(252366215ns) {ProcessConfigured:135(41484ns) {StartInitialHandshake:157(39856ns) {DropHandshakes:469(2002ns) {DropIncomingHandshake:441(423ns) DropOutgoingHandshake:456(396ns)} GenerateSessionVirtualId:988(254ns) PrepareNewSessionHandshake:476(1140ns) {SwitchToState:178(529ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.221694Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942794:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.225023Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518699335376082:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.241723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518694368720782:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.226073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518695533542068:2069] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.123050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518700632852302:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.242181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518696051975454:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(344190593ns) {Configure:104(344186555ns) {ProcessConfigured:135(15825706ns) {StartInitialHandshake:157(15822498ns) {DropHandshakes:469(2188ns) {DropIncomingHandshake:441(414ns) DropOutgoingHandshake:456(578ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:476(2505ns) {SwitchToState:178(1037ns)}}}}}
2025-11-11T17:19:32.226957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518695040408715:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.226106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518695533542068:2069] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.233751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518697761283305:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(312921480ns) {Configure:104(312915962ns) {ProcessConfigured:135(31957006ns) {StartInitialHandshake:157(31955056ns) {DropHandshakes:469(2687ns) {DropIncomingHandshake:441(577ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(576ns) PrepareNewSessionHandshake:476(3704ns) {SwitchToState:178(2123ns)}}}}}
2025-11-11T17:19:32.297708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518694368720782:2059] [node 5] ICP35 event processing took too much time PendingConnection:278(39965995ns) {IncomingHandshake:261(16107717ns) {DropIncomingHandshake:441(845ns) PrepareNewSessionHandshake:476(2884ns) {SwitchToState:178(1404ns)} GenerateSessionVirtualId:988(1102ns) IssueIncomingHandshakeReply:193(16065162ns) {SwitchToState:178(2990ns)}}}
2025-11-11T17:19:32.123251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518700632852303:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.254546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518697761283323:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(304403253ns) {Configure:104(304396874ns) {ProcessConfigured:135(60679ns) {StartInitialHandshake:157(58028ns) {DropHandshakes:469(2909ns) {DropIncomingHandshake:441(784ns) DropOutgoingHandshake:456(448ns)} GenerateSessionVirtualId:988(848ns) PrepareNewSessionHandshake:476(1954ns) {SwitchToState:178(679ns)}}}}}
2025-11-11T17:19:32.243677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942796:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.253837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518695533542068:2069] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:32.322226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688141:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.322365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688144:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.170068Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852302:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.286228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518695040408716:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(408141716ns) {Configure:104(408136614ns) {ProcessConfigured:135(47017ns) {StartInitialHandshake:157(44525ns) {DropHandshakes:469(584ns) {DropIncomingHandshake:441(89ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1467ns) {SwitchToState:178(774ns)}}}}}
2025-11-11T17:19:32.249658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942795:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.277740Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518697761283330:2069] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.278536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518699828509408:2084] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(96493950ns) {Configure:104(96487504ns) {ProcessConfigured:135(63134ns) {StartInitialHandshake:157(60229ns) {DropHandshakes:469(3562ns) {DropIncomingHandshake:441(760ns) DropOutgoingHandshake:456(1088ns)} GenerateSessionVirtualId:988(743ns) PrepareNewSessionHandshake:476(2048ns) {SwitchToState:178(626ns)}}}}}
2025-11-11T17:19:32.350241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518694368720768:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(324365959ns) {Configure:104(304540396ns) {ProcessConfigured:135(56720ns) {StartInitialHandshake:157(54356ns) {DropHandshakes:469(3118ns) {DropIncomingHandshake:441(846ns) DropOutgoingHandshake:456(592ns)} GenerateSessionVirtualId:988(455ns) PrepareNewSessionHandshake:476(1150ns) {SwitchToState:178(255ns)}}}}}
2025-11-11T17:19:32.314094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518699335376092:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-11T17:19:32.184273Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852303:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.293527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518696051975436:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(477588112ns) {Configure:104(477578219ns) {ProcessConfigured:135(7628252ns) {StartInitialHandshake:157(7625989ns) {DropHandshakes:469(3833ns) {DropIncomingHandshake:441(742ns) DropOutgoingHandshake:456(1299ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:476(2688ns) {SwitchToState:178(1077ns)}}}}}
2025-11-11T17:19:32.238038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518700632852306:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:19:32.281906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518702056250676:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.278710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509412:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.322677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518699335376086:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.293880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518696051975436:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.393820Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518698663688141:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.393962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518694368720769:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.396565Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.279090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509418:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.374141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518697761283328:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(204103534ns) {Configure:104(204095678ns) {ProcessConfigured:135(4210499ns) {StartInitialHandshake:157(4206876ns) {DropHandshakes:469(1815ns) {DropIncomingHandshake:441(468ns) DropOutgoingHandshake:456(242ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:476(2122ns) {SwitchToState:178(749ns)}}}}}
2025-11-11T17:19:32.374613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518702056250680:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
2025-11-11T17:19:32.389953Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518702056250676:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.322699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518699335376086:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.397425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518694368720782:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518698663688134:2112] held: no
2025-11-11T17:19:32.282061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518696337884989:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(348968375ns) {Configure:104(348960147ns) {ProcessConfigured:135(54573ns) {StartInitialHandshake:157(52183ns) {DropHandshakes:469(1718ns) {DropIncomingHandshake:441(446ns) DropOutgoingHandshake:456(274ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:476(2597ns) {SwitchToState:178(787ns)}}}}}
2025-11-11T17:19:32.426020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518698663688144:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.314399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518700346942817:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.409967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518702056250683:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.323468Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518699335376086:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:32.285798Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852299:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.294495Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518699828509413:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.302254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518695533542041:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.323671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518695040408755:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.334954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518696051975437:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(496414664ns) {Configure:104(496409287ns) {ProcessConfigured:135(31198375ns) {StartInitialHandshake:157(31195759ns) {DropHandshakes:469(3122ns) {DropIncomingHandshake:441(715ns) DropOutgoingHandshake:456(696ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:476(3328ns) {SwitchToState:178(1519ns)}}}}}
2025-11-11T17:19:32.429136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688142:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.323725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:32.421799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518697761283306:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.285835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518700632852299:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.337708Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518699828509412:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.453653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688143:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.374777Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518700346942796:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.421901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518702056250674:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.323769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518695040408755:2072] [node 5] ICP22 created new session: [2:7571518699335376094:2072]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.298942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852301:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.337834Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518699828509418:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.401570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518696051975436:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.455277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518694368720767:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.323806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518699335376094:2072] [node 5] ICS09 handshake done sender: [2:7571518699335376086:2114] self: [2:7571518699335376085:0] peer: [5:7571518699828509420:0] socket: 319 qp: -1
2025-11-11T17:19:32.323837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518699335376094:2072] [node 5] ICS10 traffic start
2025-11-11T17:19:32.323868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:32.330146Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518699335376082:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:32.346069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518695040408715:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.346130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:32.346181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518695040408715:2048] [node 1] ICP22 created new session: [2:7571518699335376098:2048]
2025-11-11T17:19:32.346230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518699335376098:2048] [node 1] ICS09 handshake done sender: [2:7571518699335376082:2113] self: [2:7571518699335376089:0] peer: [1:7571518700632852298:0] socket: 316 qp: -1
2025-11-11T17:19:32.346262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518699335376098:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:32.346304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:32.435945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518699335376103:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-11T17:19:32.442115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518699335376102:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-11T17:19:32.449796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518695040408761:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(379889584ns) {Configure:104(379881781ns) {ProcessConfigured:135(23577434ns) {StartInitialHandshake:157(23574246ns) {DropHandshakes:469(2017ns) {DropIncomingHandshake:441(363ns) DropOutgoingHandshake:456(537ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:476(2918ns) {SwitchToState:178(881ns)}}}}}
2025-11-11T17:19:32.450810Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518699335376102:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.490330Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518699335376103:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.490420Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518699335376081:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.490447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518699335376081:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.491774Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518699335376081:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:32.493550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518695040408760:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.498373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:32.509650Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518699335376091:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.509687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518699335376091:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.421929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518702056250674:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.298963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518700632852301:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.338026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518695533542040:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(445331874ns) {Configure:104(445326421ns) {ProcessConfigured:135(17366006ns) {StartInitialHandshake:157(17364176ns) {DropHandshakes:469(3260ns) {DropIncomingHandshake:441(712ns) DropOutgoingHandshake:456(680ns)} GenerateSessionVirtualId:988(1043ns) PrepareNewSessionHandshake:476(2345ns) {SwitchToState:178(1151ns)}}}}}
2025-11-11T17:19:32.426427Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518702056250674:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:32.462072Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518698663688134:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:32.405978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942798:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.537852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518695040408760:2069] [node 6] ICP22 created new session: [2:7571518699335376106:2069]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.357995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518695533542042:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.301080Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518700632852299:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:32.406004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518700346942798:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.537930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518699335376106:2069] [node 6] ICS09 handshake done sender: [2:7571518699335376081:2112] self: [2:7571518699335376080:0] peer: [6:7571518698663688153:0] socket: 318 qp: -1
2025-11-11T17:19:32.462323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518694368720782:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.413961Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518700346942798:2109] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:32.414278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518696051975467:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.414337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:32.414383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518696051975467:2067] [node 5] ICP22 created new session: [3:7571518700346942821:2067]
2025-11-11T17:19:32.414434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518700346942821:2067] [node 5] ICS09 handshake done sender: [3:7571518700346942798:2109] self: [3:7571518700346942797:0] peer: [5:7571518699828509422:0] socket: 313 qp: -1
2025-11-11T17:19:32.414461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518700346942821:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:32.414496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:32.422862Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942800:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.422901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518700346942800:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.427945Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942808:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.427968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518700346942808:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.433896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:32.438356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518696051975436:2048] [node 1] ICP22 created new session: [3:7571518700346942824:2048]
2025-11-11T17:19:32.442350Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518700346942808:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:32.442839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518696051975454:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.442891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:32.442942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518696051975454:2064] [node 4] ICP22 created new session: [3:7571518700346942826:2064]
2025-11-11T17:19:32.442980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518700346942826:2064] [node 4] ICS09 handshake done sender: [3:7571518700346942808:2115] self: [3:7571518700346942807:0] peer: [4:7571518702056250682:0] socket: 326 qp: -1
2025-11-11T17:19:32.443015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518700346942826:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:32.443054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:32.444089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518700346942825:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-11T17:19:32.450696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518700346942818:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-11T17:19:32.455378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518700346942824:2048] [node 1] ICS09 handshake done sender: [3:7571518700346942796:2108] self: [3:7571518700346942816:0] peer: [1:7571518700632852300:0] socket: 312 qp: -1
2025-11-11T17:19:32.464269Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942825:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.465672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518700346942824:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:32.465807Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518700346942800:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:32.470043Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518700346942818:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.470152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518696051975437:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.477731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:32.509357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518700346942779:2073] [node 6] ICP35 event processing took too much time PendingConnection:278(22572763ns) {IncomingHandshake:261(22571000ns) {DropIncomingHandshake:441(2053ns) PrepareNewSessionHandshake:476(3731ns) {SwitchToState:178(1543ns)} GenerateSessionVirtualId:988(444ns) IssueIncomingHandshakeReply:193(8191ns)}}
2025-11-11T17:19:32.509424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518700346942779:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.517724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:32.539107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518696051975436:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(145095124ns) {HandleHandshakeStatus:329(145091964ns) {DropIncomingHandshake:441(1344ns) DropOutgoingHandshake:456(6102ns) SwitchToState:178(344ns) ProcessPendingSessionEvents:495(21120578ns) {ForwardSessionEventToSession:612(21068358ns) ForwardSessionEventToSession:612(4129ns) ForwardSessionEventToSession:612(3071ns) ForwardSessionEventToSession:612(11846ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(2529ns) ForwardSessionEventToSession:612(5111ns) ForwardSessionEventToSession:612(4676ns) ForwardSessionEventToSession:612(4363ns) ForwardSessionEventToSession:612(3026ns)}}}
2025-11-11T17:19:32.547157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518700346942779:2073] [node 6] ICP22 created new session: [3:7571518700346942832:2073]
2025-11-11T17:19:32.558095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518700346942833:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-11T17:19:32.565657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518700346942832:2073] [node 6] ICS09 handshake done sender: [3:7571518700346942800:2110] self: [3:7571518700346942799:0] peer: [6:7571518698663688147:0] socket: 314 qp: -1
2025-11-11T17:19:32.574704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518700346942832:2073] [node 6] ICS10 traffic start
2025-11-11T17:19:32.592249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.434594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518697761283323:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.434633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:32.434675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518697761283323:2062] [node 5] ICP22 created new session: [4:7571518702056250686:2062]
2025-11-11T17:19:32.434714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518702056250686:2062] [node 5] ICS09 handshake done sender: [4:7571518702056250674:2114] self: [4:7571518702056250673:0] peer: [5:7571518699828509424:0] socket: 325 qp: -1
2025-11-11T17:19:32.434749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518702056250686:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:32.434805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:32.435461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518702056250685:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
2025-11-11T17:19:32.441684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518697761283306:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(23023786ns) {IncomingHandshake:261(23021748ns) {DropIncomingHandshake:441(464ns) PrepareNewSessionHandshake:476(1058ns) {SwitchToState:178(495ns)} GenerateSessionVirtualId:988(330ns) IssueIncomingHandshakeReply:193(62837ns) {SwitchToState:178(768ns)}}}
2025-11-11T17:19:32.449935Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.450341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518697761283306:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518702056250676:2115] held: no
2025-11-11T17:19:32.457680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518702056250680:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.481724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518697761283305:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.497708Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518702056250676:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:32.505901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518702056250683:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.506463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518697761283323:2062] [node 5] ICP18 handshake (actor [4:7571518702056250683:2118]) is too old
2025-11-11T17:19:32.506566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518702056250690:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
2025-11-11T17:19:32.545884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518697761283306:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.547455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:32.560125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518697761283306:2050] [node 3] ICP22 created new session: [4:7571518702056250692:2050]
2025-11-11T17:19:32.569092Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518702056250690:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.569348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518702056250693:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
2025-11-11T17:19:32.569423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518697761283304:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.586429Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518702056250680:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:32.586585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518697761283305:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.586636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:32.586674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518697761283305:2049] [node 2] ICP22 created new session: [4:7571518702056250695:2049]
2025-11-11T17:19:32.586730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518702056250695:2049] [node 2] ICS09 handshake done sender: [4:7571518702056250680:2117] self: [4:7571518702056250689:0] peer: [2:7571518699335376097:0] socket: 340 qp: -1
2025-11-11T17:19:32.586764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518702056250695:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:32.586800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:32.589839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518702056250692:2050] [node 3] ICS09 handshake done sender: [4:7571518702056250676:2115] self: [4:7571518702056250682:0] peer: [3:7571518700346942807:0] socket: 327 qp: -1
2025-11-11T17:19:32.597840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518702056250692:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:32.606127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:32.606388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518697761283306:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(71739472ns) {HandleHandshakeStatus:329(71736366ns) {DropIncomingHandshake:441(3329ns) DropOutgoingHandshake:456(931ns) SwitchToState:178(956ns) ProcessPendingSessionEvents:495(988ns)}}
2025-11-11T17:19:32.311206Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852306:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.365784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518695533542066:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.462372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:32.605994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518700346942818:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:32.318147Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518700632852301:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:32.537969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518699335376106:2069] [node 6] ICS10 traffic start
2025-11-11T17:19:32.462423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518694368720782:2059] [node 5] ICP22 created new session: [6:7571518698663688150:2059]
2025-11-11T17:19:32.462466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518698663688150:2059] [node 5] ICS09 handshake done sender: [6:7571518698663688134:2112] self: [6:7571518698663688140:0] peer: [5:7571518695533542067:0] socket: 303 qp: -1
2025-11-11T17:19:32.462504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518698663688150:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:32.462535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:32.469905Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518698663688142:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.473996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688152:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.610389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518696051975437:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.610433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:32.610480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518696051975437:2049] [node 2] ICP22 created new session: [3:7571518700346942835:2049]
2025-11-11T17:19:32.610532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518700346942835:2049] [node 2] ICS09 handshake done sender: [3:7571518700346942818:2119] self: [3:7571518700346942829:0] peer: [2:7571518699335376090:0] socket: 339 qp: -1
2025-11-11T17:19:32.610571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518700346942835:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:32.610604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:32.654073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518700346942779:2073] [node 6] ICP35 event processing took too much time PendingConnection:278(144368846ns) {HandleHandshakeStatus:329(144365870ns) {DropIncomingHandshake:441(8235893ns) DropOutgoingHandshake:456(841ns) SwitchToState:178(512ns) ProcessPendingSessionEvents:495(15506949ns) {ForwardSessionEventToSession:612(15449423ns) ForwardSessionEventToSession:612(11685ns) ForwardSessionEventToSession:612(3556ns) ForwardSessionEventToSession:612(2987ns) ForwardSessionEventToSession:612(4495ns) ForwardSessionEventToSession:612(2685ns) ForwardSessionEventToSession:612(9375ns) ForwardSessionEventToSession:612(2475ns) ForwardSessionEventToSession:612(2681ns) ForwardSessionEventToSession:612(5115ns)}}}
2025-11-11T17:19:32.365848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.538032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:32.481701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518694368720768:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.491555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688154:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.492229Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518698663688142:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:32.492421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518694368720768:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.492474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:32.492520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518694368720768:2049] [node 2] ICP22 created new session: [6:7571518698663688155:2049]
2025-11-11T17:19:32.492560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518698663688155:2049] [node 2] ICS09 handshake done sender: [6:7571518698663688142:2116] self: [6:7571518698663688153:0] peer: [2:7571518699335376080:0] socket: 324 qp: -1
2025-11-11T17:19:32.492592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518698663688155:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:32.492628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:32.514314Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518698663688141:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:32.557862Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.567905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518694368720769:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.567953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:32.567999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518694368720769:2050] [node 3] ICP22 created new session: [6:7571518698663688157:2050]
2025-11-11T17:19:32.568049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518698663688157:2050] [node 3] ICS09 handshake done sender: [6:7571518698663688141:2115] self: [6:7571518698663688147:0] peer: [3:7571518700346942799:0] socket: 321 qp: -1
2025-11-11T17:19:32.568081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518698663688157:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:32.568111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:32.568322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518694368720769:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:32.571935Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.572811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518694368720767:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518698663688144:2118] held: no
2025-11-11T17:19:32.593977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688159:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.594469Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518698663688144:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:32.594670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518694368720767:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.594723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:32.594764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518694368720767:2048] [node 1] ICP22 created new session: [6:7571518698663688160:2048]
2025-11-11T17:19:32.594806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518698663688160:2048] [node 1] ICS09 handshake done sender: [6:7571518698663688144:2118] self: [6:7571518698663688149:0] peer: [1:7571518700632852308:0] socket: 333 qp: -1
2025-11-11T17:19:32.594838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518698663688160:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:32.594870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:32.637932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688162:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.669721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518698663688162:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.673996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518702056250698:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21026
2025-11-11T17:19:32.329752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518696337884993:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(169114765ns) {Configure:104(169110048ns) {ProcessConfigured:135(56010872ns) {StartInitialHandshake:157(51096093ns) {DropHandshakes:469(4021ns) {DropIncomingHandshake:441(502ns) DropOutgoingHandshake:456(580ns)} GenerateSessionVirtualId:988(2359ns) PrepareNewSessionHandshake:476(2215ns) {SwitchToState:178(845ns)}}}}}
2025-11-11T17:19:32.381701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518695533542066:2068] [node 6] ICP22 created new session: [5:7571518699828509423:2068]
2025-11-11T17:19:32.538532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518695040408760:2069] [node 6] ICP35 event processing took too much time PendingConnection:278(44747689ns) {HandleHandshakeStatus:329(44745779ns) {DropIncomingHandshake:441(166ns) DropOutgoingHandshake:456(343ns) SwitchToState:178(435ns) ProcessPendingSessionEvents:495(113625ns) {ForwardSessionEventToSession:612(61796ns) ForwardSessionEventToSession:612(3164ns) ForwardSessionEventToSession:612(5894ns) ForwardSessionEventToSession:612(4037ns) ForwardSessionEventToSession:612(2934ns) ForwardSessionEventToSession:612(8035ns) ForwardSessionEventToSession:612(3059ns) ForwardSessionEventToSession:612(6252ns) ForwardSessionEventToSession:612(5047ns) ForwardSessionEventToSession:612(2786ns)}}}
2025-11-11T17:19:32.545639Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518699335376100:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.545690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518699335376100:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.559537Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518699335376091:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:32.585945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518699335376100:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:32.598103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518695040408716:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.607244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:32.625809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518695040408761:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.625875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:32.638195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518695040408716:2049] [node 3] ICP22 created new session: [2:7571518699335376111:2049]
2025-11-11T17:19:32.645725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518699335376111:2049] [node 3] ICS09 handshake done sender: [2:7571518699335376091:2115] self: [2:7571518699335376090:0] peer: [3:7571518700346942829:0] socket: 337 qp: -1
2025-11-11T17:19:32.645752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518695040408761:2067] [node 4] ICP22 created new session: [2:7571518699335376112:2067]
2025-11-11T17:19:32.659071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518699335376111:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:32.659120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:32.659912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518699335376112:2067] [node 4] ICS09 handshake done sender: [2:7571518699335376100:2117] self: [2:7571518699335376097:0] peer: [4:7571518702056250689:0] socket: 338 qp: -1
2025-11-11T17:19:32.662155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518699335376112:2067] [node 4] ICS10 traffic start
2025-11-11T17:19:32.667725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:32.694286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518695040408761:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(68252208ns) {HandleHandshakeStatus:329(68248620ns) {DropIncomingHandshake:441(15058ns) DropOutgoingHandshake:456(414ns) SwitchToState:178(650ns) ProcessPendingSessionEvents:495(120140ns) {ForwardSessionEventToSession:612(74037ns) ForwardSessionEventToSession:612(3856ns) ForwardSessionEventToSession:612(8061ns) ForwardSessionEventToSession:612(2936ns) ForwardSessionEventToSession:612(2747ns) ForwardSessionEventToSession:612(5018ns) ForwardSessionEventToSession:612(2980ns) ForwardSessionEventToSession:612(7601ns) ForwardSessionEventToSession:612(2174ns) ForwardSessionEventToSession:612(2688ns)}}}
2025-11-11T17:19:32.358213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518696337884937:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.698095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518694368720783:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.358262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:32.386315Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63490) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.706188Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.706010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518694368720783:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(16184074ns) {IncomingHandshake:261(8356852ns) {DropIncomingHandshake:441(1000ns) PrepareNewSessionHandshake:476(2941ns) {SwitchToState:178(1453ns)} GenerateSessionVirtualId:988(1217ns) IssueIncomingHandshakeReply:193(65913ns) {SwitchToState:178(644ns)}}}
2025-11-11T17:19:32.745932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518695040408716:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(135874006ns) {HandleHandshakeStatus:329(135871057ns) {DropIncomingHandshake:441(13594ns) DropOutgoingHandshake:456(3989ns) SwitchToState:178(691ns) ProcessPendingSessionEvents:495(40304286ns) {ForwardSessionEventToSession:612(19992091ns) ForwardSessionEventToSession:612(8919ns) ForwardSessionEventToSession:612(6837676ns) ForwardSessionEventToSession:612(4057ns) ForwardSessionEventToSession:612(2214ns) ForwardSessionEventToSession:612(5318159ns) ForwardSessionEventToSession:612(3345ns) ForwardSessionEventToSession:612(7814855ns) ForwardSessionEventToSession:612(298718ns) ForwardSessionEventToSession:612(4303ns)}}}
2025-11-11T17:19:32.395077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518699828509408:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.395642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518695533542041:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63490) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518699828509413:2115] held: no
2025-11-11T17:19:32.396280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509425:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.401674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518699828509423:2068] [node 6] ICS09 handshake done sender: [5:7571518695533542068:2069] self: [5:7571518695533542067:0] peer: [6:7571518698663688140:0] socket: 302 qp: -1
2025-11-11T17:19:32.413688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518699828509423:2068] [node 6] ICS10 traffic start
2025-11-11T17:19:32.427198Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.441733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:32.445787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518699828509425:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.474482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518695533542042:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518699828509412:2114] held: no
2025-11-11T17:19:32.475019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509421:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.507281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518695533542040:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:32.507341Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518699828509413:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:32.513686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509430:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.526002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518695533542041:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.526056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:32.526108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518695533542041:2049] [node 2] ICP22 created new session: [5:7571518699828509432:2049]
2025-11-11T17:19:32.526157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518699828509432:2049] [node 2] ICS09 handshake done sender: [5:7571518699828509413:2115] self: [5:7571518699828509420:0] peer: [2:7571518699335376085:0] socket: 320 qp: -1
2025-11-11T17:19:32.526195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518699828509432:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:32.526234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:32.546292Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8786) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.546339Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518699828509418:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:32.561795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518699828509408:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8786) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518699828509418:2117] held: no
2025-11-11T17:19:32.601725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509428:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.606920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518695533542066:2068] [node 6] ICP35 event processing took too much time PendingConnection:278(202096790ns) {HandleHandshakeStatus:329(202094494ns) {DropIncomingHandshake:441(15937ns) DropOutgoingHandshake:456(562ns) SwitchToState:178(469ns) ProcessPendingSessionEvents:495(76329393ns) {ForwardSessionEventToSession:612(31855244ns) ForwardSessionEventToSession:612(8242ns) ForwardSessionEventToSession:612(3075ns) ForwardSessionEventToSession:612(28011751ns) ForwardSessionEventToSession:612(5549ns) ForwardSessionEventToSession:612(3888637ns) ForwardSessionEventToSession:612(12064598ns) ForwardSessionEventToSession:612(4306ns) ForwardSessionEventToSession:612(453919ns) ForwardSessionEventToSession:612(3744ns)}}}
2025-11-11T17:19:32.614306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518699828509434:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47356
2025-11-11T17:19:32.626058Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518699828509412:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:32.638590Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:32.666081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518695533542040:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518699828509425:2120] held: no
2025-11-11T17:19:32.666186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518695533542042:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.666243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:32.666284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518695533542042:2050] [node 3] ICP22 created new session: [5:7571518699828509435:2050]
2025-11-11T17:19:32.666336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518699828509435:2050] [node 3] ICS09 handshake done sender: [5:7571518699828509412:2114] self: [5:7571518699828509422:0] peer: [3:7571518700346942797:0] socket: 315 qp: -1
2025-11-11T17:19:32.666364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518699828509435:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:32.666396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:32.666996Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518699828509425:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:32.697770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518699828509408:2084] [node 4] ICP35 event processing took too much time PendingConnection:278(24187774ns) {HandleHandshakeStatus:395(24184031ns) {LogHandshakeFail:472(13148ns) {UpdateErrorStateLog:544(5615ns)} DropOutgoingHandshake:456(1201ns) ProcessPendingSessionEvents:495(12115708ns) {DropSessionEvent:512(12006ns) DropSessionEvent:512(4147ns) DropSessionEvent:512(5029ns) DropSessionEvent:512(404802ns) DropSessionEvent:512(3261ns) DropSessionEvent:512(11662710ns) DropSessionEvent:512(4572ns) DropSessionEvent:512(4241ns) DropSessionEvent:512(4929ns) DropSessionEvent:512(4462ns)}}}
2025-11-11T17:19:32.697834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518695533542040:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.701810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518699828509408:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.701884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:32.701931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518699828509408:2084] [node 4] ICP22 created new session: [5:7571518699828509437:2084]
2025-11-11T17:19:32.701980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518699828509437:2084] [node 4] ICS09 handshake done sender: [5:7571518699828509418:2117] self: [5:7571518699828509424:0] peer: [4:7571518702056250673:0] socket: 329 qp: -1
2025-11-11T17:19:32.702018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518699828509437:2084] [node 4] ICS10 traffic start
2025-11-11T17:19:32.702057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:32.720434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:32.725736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518695533542040:2048] [node 1] ICP22 created new session: [5:7571518699828509439:2048]
2025-11-11T17:19:32.749865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518699828509439:2048] [node 1] ICS09 handshake done sender: [5:7571518699828509425:2120] self: [5:7571518699828509431:0] peer: [1:7571518700632852307:0] socket: 290 qp: -1
2025-11-11T17:19:32.749920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518699828509439:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:32.358321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518696337884937:2049] [node 3] ICP22 created new session: [1:7571518700632852318:2049]
2025-11-11T17:19:32.707222Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518702056250690:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:32.757947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518698663688164:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30568
2025-11-11T17:19:32.358369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518700632852318:2049] [node 3] ICS09 handshake done sender: [1:7571518700632852301:2111] self: [1:7571518700632852300:0] peer: [3:7571518700346942816:0] socket: 310 qp: -1
2025-11-11T17:19:32.718113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518697761283304:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518702056250690:2120] held: no
2025-11-11T17:19:32.358404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518700632852318:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:32.718220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518697761283304:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.718265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:32.358439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:32.718306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518697761283304:2048] [node 1] ICP22 created new session: [4:7571518702056250699:2048]
2025-11-11T17:19:32.358895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518696337884936:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.718366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518702056250699:2048] [node 1] ICS09 handshake done sender: [4:7571518702056250690:2120] self: [4:7571518702056250694:0] peer: [1:7571518700632852315:0] socket: 355 qp: -1
2025-11-11T17:19:32.358940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:32.718401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518702056250699:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:32.358972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518696337884936:2048] [node 2] ICP22 created new session: [1:7571518700632852320:2048]
2025-11-11T17:19:32.359011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518700632852320:2048] [node 2] ICS09 handshake done sender: [1:7571518700632852299:2110] self: [1:7571518700632852298:0] peer: [2:7571518699335376089:0] socket: 309 qp: -1
2025-11-11T17:19:32.718457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:32.758264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:32.758543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518695533542040:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(60615248ns) {HandleHandshakeStatus:329(60611938ns) {DropIncomingHandshake:441(923ns) DropOutgoingHandshake:456(815ns) SwitchToState:178(629ns) ProcessPendingSessionEvents:495(940ns)}}
2025-11-11T17:19:32.359036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518700632852320:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:32.718868Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518702056250679:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.718893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518702056250679:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.729672Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518702056250679:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:32.781735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518697761283328:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.781816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:32.359058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:32.789997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518697761283328:2059] [node 6] ICP22 created new session: [4:7571518702056250702:2059]
2025-11-11T17:19:32.391497Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518696337884970:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(441972362ns) {Configure:104(441964701ns) {ProcessConfigured:135(25972187ns) {StartInitialHandshake:157(25969001ns) {DropHandshakes:469(5943177ns) {DropIncomingHandshake:441(615ns) DropOutgoingHandshake:456(1077ns)} GenerateSessionVirtualId:988(1846ns) PrepareNewSessionHandshake:476(4131ns) {SwitchToState:178(2147ns)}}}}}
2025-11-11T17:19:32.418046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518700632852323:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:19:32.493916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852309:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.493953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518700632852309:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.513955Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852323:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.533977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518700632852309:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:32.559970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852310:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.560006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518700632852310:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.566905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518696337884989:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.566967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:32.567018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518696337884989:2075] [node 6] ICP22 created new session: [1:7571518700632852328:2075]
2025-11-11T17:19:32.567074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518700632852328:2075] [node 6] ICS09 handshake done sender: [1:7571518700632852309:2115] self: [1:7571518700632852308:0] peer: [6:7571518698663688149:0] socket: 332 qp: -1
2025-11-11T17:19:32.567110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518700632852328:2075] [node 6] ICS10 traffic start
2025-11-11T17:19:32.567149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:32.570648Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518700632852310:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:32.570863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518696337884993:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.570906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:32.570948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518696337884993:2076] [node 5] ICP22 created new session: [1:7571518700632852331:2076]
2025-11-11T17:19:32.570987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518700632852331:2076] [node 5] ICS09 handshake done sender: [1:7571518700632852310:2116] self: [1:7571518700632852307:0] peer: [5:7571518699828509431:0] socket: 317 qp: -1
2025-11-11T17:19:32.571015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518700632852331:2076] [node 5] ICS10 traffic start
2025-11-11T17:19:32.571042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:32.641696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518700632852317:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:32.641739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518700632852317:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:32.662250Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518700632852317:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:32.680452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518696337884970:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:32.680551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:32.680601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518696337884970:2074] [node 4] ICP22 created new session: [1:7571518700632852334:2074]
2025-11-11T17:19:32.680664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518700632852334:2074] [node 4] ICS09 handshake done sender: [1:7571518700632852317:2117] self: [1:7571518700632852315:0] peer: [4:7571518702056250694:0] socket: 305 qp: -1
2025-11-11T17:19:32.680700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518700632852334:2074] [node 4] ICS10 traffic start
2025-11-11T17:19:32.680744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:32.773684Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518698663688162:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:32.797992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518694368720783:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:32.798057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:32.798106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518694368720783:2061] [node 4] ICP22 created new session: [6:7571518698663688165:2061]
2025-11-11T17:19:32.798149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518698663688165:2061] [node 4] ICS09 handshake done sender: [6:7571518698663688162:2123] self: [6:7571518698663688163:0] peer: [4:7571518702056250678:0] socket: 297 qp: -1
2025-11-11T17:19:32.798184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518698663688165:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:32.798232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:32.801681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518702056250702:2059] [node 6] ICS09 handshake done sender: [4:7571518702056250679:2116] self: [4:7571518702056250678:0] peer: [6:7571518698663688163:0] socket: 357 qp: -1
2025-11-11T17:19:32.813887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518702056250702:2059] [node 6] ICS10 traffic start
2025-11-11T17:19:32.813945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:32.814437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518697761283328:2059] [node 6] ICP35 event processing took too much time PendingConnection:278(32478821ns) {HandleHandshakeStatus:329(32474734ns) {DropIncomingHandshake:441(27306ns) DropOutgoingHandshake:456(533ns) SwitchToState:178(678ns) ProcessPendingSessionEvents:495(102499ns) {ForwardSessionEventToSession:612(53877ns) ForwardSessionEventToSession:612(3130ns) ForwardSessionEventToSession:612(5086ns) ForwardSessionEventToSession:612(4845ns) ForwardSessionEventToSession:612(4848ns) ForwardSessionEventToSession:612(2511ns) ForwardSessionEventToSession:612(3301ns) ForwardSessionEventToSession:612(5302ns) ForwardSessionEventToSession:612(2894ns) ForwardSessionEventToSession:612(2322ns)}}}
2025-11-11T17:19:32.900145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:32.900384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518699335376098:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-11T17:19:32.900428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518695040408715:2048] [node 1] ICP30 unregister session Session# [2:7571518699335376098:2048] VirtualId# [2:7571518699335376089:0]
2025-11-11T17:19:32.900458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518699335376098:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:32.900544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:32.900582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:32.903222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:32.903430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518700346942824:2048] [node 1] ICS07 socket disconnect 312 reason# EndOfStream
2025-11-11T17:19:32.903463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518700346942824:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:32.903539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:32.903577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518700346942824:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:32.913708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:32.913934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518698663688160:2048] [node 1] ICS01 socket: 333 reason# EndOfStream
2025-11-11T17:19:32.913970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518694368720767:2048] [node 1] ICP30 unregister session Session# [6:7571518698663688160:2048] VirtualId# [6:7571518698663688149:0]
2025-11-11T17:19:32.914003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518698663688160:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:32.914113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:32.914172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:32.909689Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:19:32.913708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518696051975436:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:19:32.913748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518700346942824:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:32.914417Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Couldn't connect to any resolved address
2025-11-11T17:19:32.914646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518696051975436:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:32.914690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518700346942824:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:32.914722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518696051975436:2048] [node 1] ICP30 unregister session Session# [3:7571518700346942824:2048] VirtualId# [3:7571518700346942816:0]
2025-11-11T17:19:32.914761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:32.914791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518696051975436:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:32.914807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:32.917772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:32.917964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518702056250699:2048] [node 1] ICS01 socket: 355 reason# EndOfStream
2025-11-11T17:19:32.918004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518697761283304:2048] [node 1] ICP30 unregister session Session# [4:7571518702056250699:2048] VirtualId# [4:7571518702056250694:0]
2025-11-11T17:19:32.918031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518702056250699:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:32.918112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:32.918147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:32.921716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:32.921932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518699828509439:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-11T17:19:32.921964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518695533542040:2048] [node 1] ICP30 unregister session Session# [5:7571518699828509439:2048] VirtualId# [5:7571518699828509431:0]
2025-11-11T17:19:32.921993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518699828509439:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:32.922105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:32.922164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:32.925706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518696051975436:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:33.004956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:33.005175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518700346942835:2049] [node 2] ICS01 socket: 339 reason# EndOfStream
2025-11-11T17:19:33.005220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518696051975437:2049] [node 2] ICP30 unregister session Session# [3:7571518700346942835:2049] VirtualId# [3:7571518700346942829:0]
2025-11-11T17:19:33.005250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518700346942835:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.005336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:33.005380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:33.013760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:33.014010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518699828509432:2049] [node 2] ICS01 socket: 320 reason# EndOfStream
2025-11-11T17:19:33.014041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518695533542041:2049] [node 2] ICP30 unregister session Session# [5:7571518699828509432:2049] VirtualId# [5:7571518699828509420:0]
2025-11-11T17:19:33.014068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518699828509432:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.014137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:33.013870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:33.014172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:33.014093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518698663688155:2049] [node 2] ICS01 socket: 324 reason# EndOfStream
2025-11-11T17:19:33.014128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518694368720768:2049] [node 2] ICP30 unregister session Session# [6:7571518698663688155:2049] VirtualId# [6:7571518698663688153:0]
2025-11-11T17:19:33.014155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518698663688155:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.014244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:33.014288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:33.013760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:33.014005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518702056250695:2049] [node 2] ICS01 socket: 340 reason# EndOfStream
2025-11-11T17:19:33.014040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518697761283305:2049] [node 2] ICP30 unregister session Session# [4:7571518702056250695:2049] VirtualId# [4:7571518702056250689:0]
2025-11-11T17:19:33.014070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518702056250695:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.014137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:33.014171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:33.102850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:33.103092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518702056250692:2050] [node 3] ICS01 socket: 327 reason# EndOfStream
2025-11-11T17:19:33.103128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518697761283306:2050] [node 3] ICP30 unregister session Session# [4:7571518702056250692:2050] VirtualId# [4:7571518702056250682:0]
2025-11-11T17:19:33.103161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518702056250692:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.103240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:33.103276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:33.109750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:33.109989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518699828509435:2050] [node 3] ICS01 socket: 315 reason# EndOfStream
2025-11-11T17:19:33.110026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518695533542042:2050] [node 3] ICP30 unregister session Session# [5:7571518699828509435:2050] VirtualId# [5:7571518699828509422:0]
2025-11-11T17:19:33.110057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518699828509435:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.110143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:33.110186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:33.110280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:33.110447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518698663688157:2050] [node 3] ICS01 socket: 321 reason# EndOfStream
2025-11-11T17:19:33.110478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518694368720769:2050] [node 3] ICP30 unregister session Session# [6:7571518698663688157:2050] VirtualId# [6:7571518698663688147:0]
2025-11-11T17:19:33.110499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518698663688157:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.110554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:33.110589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:33.229710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:33.229950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518699828509437:2084] [node 4] ICS01 socket: 329 reason# EndOfStream
2025-11-11T17:19:33.229989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518699828509408:2084] [node 4] ICP30 unregister session Session# [5:7571518699828509437:2084] VirtualId# [5:7571518699828509424:0]
2025-11-11T17:19:33.230023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518699828509437:2084] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.230114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:33.230154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:33.230244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:33.230405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518698663688165:2061] [node 4] ICS01 socket: 297 reason# EndOfStream
2025-11-11T17:19:33.230433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518694368720783:2061] [node 4] ICP30 unregister session Session# [6:7571518698663688165:2061] VirtualId# [6:7571518698663688163:0]
2025-11-11T17:19:33.230539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518698663688165:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.230704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:33.230772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:33.297754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:33.297986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518698663688150:2059] [node 5] ICS01 socket: 303 reason# EndOfStream
2025-11-11T17:19:33.298026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518694368720782:2059] [node 5] ICP30 unregister session Session# [6:7571518698663688150:2059] VirtualId# [6:7571518698663688140:0]
2025-11-11T17:19:33.298063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518698663688150:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:33.298172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:33.298217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:33.493898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518703441395511:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:33.493944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518703441395512:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:33.569903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518703332913373:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:33.569951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518703332913374:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:33.625983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518703389191330:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:33.626026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518703389191331:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:33.734068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518704091690400:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:33.734116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518704091690401:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:33.734147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518704091690402:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:33.842688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518704921015693:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:33.842740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518704921015694:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:33.842766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518704921015695:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:33.959809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518706969128869:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:33.959855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518706969128870:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:33.959882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518706969128871:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:34.202363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518708386657715:2064] [node 6] ICP01 ready to work
2025-11-11T17:19:34.207794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518707627880718:2099] [node 4] ICP01 ready to work
2025-11-11T17:19:34.244571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518708386657720:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:34.221569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518707684158644:2065] [node 4] ICP01 ready to work
2025-11-11T17:19:34.253954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518707736362860:2093] [node 6] ICP01 ready to work
2025-11-11T17:19:34.273975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518707627880723:2091] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.329062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983009:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.266334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518707684158645:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.344433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657724:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.352869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657729:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
2025-11-11T17:19:34.321729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518711264096182:2063] [node 4] ICP01 ready to work
2025-11-11T17:19:34.285855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518707627880724:2092] [node 6] ICP01 ready to work
2025-11-11T17:19:34.330485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518703332913374:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(152010239ns) {Configure:104(152006764ns) {ProcessConfigured:135(23681696ns) {StartInitialHandshake:157(23678864ns) {DropHandshakes:469(2888ns) {DropIncomingHandshake:441(717ns) DropOutgoingHandshake:456(576ns)} GenerateSessionVirtualId:988(957ns) PrepareNewSessionHandshake:476(2863ns) {SwitchToState:178(1319ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.369706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518707736362871:2092] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.321661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518709215983007:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:34.331774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518707684158652:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.508448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657768:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
2025-11-11T17:19:34.337723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518711264096183:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:34.337904Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518709215983009:2066] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.381706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518707684158653:2067] [node 6] ICP01 ready to work
2025-11-11T17:19:34.351990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518707627880718:2099] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(143922267ns) {Configure:104(143918080ns) {ProcessConfigured:135(69890821ns) {StartInitialHandshake:157(69888537ns) {DropHandshakes:469(3217ns) {DropIncomingHandshake:441(752ns) DropOutgoingHandshake:456(655ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:476(2944ns) {SwitchToState:178(69ns)}}}}}
2025-11-11T17:19:34.369731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518707736362875:2094] [node 4] ICP01 ready to work
2025-11-11T17:19:34.386892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096189:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.522529Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518708386657768:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.488793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518707627880724:2092] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(130757393ns) {Configure:104(130749933ns) {ProcessConfigured:135(67270ns) {StartInitialHandshake:157(64958ns) {DropHandshakes:469(3001ns) {DropIncomingHandshake:441(787ns) DropOutgoingHandshake:456(561ns)} GenerateSessionVirtualId:988(926ns) PrepareNewSessionHandshake:476(2018ns) {SwitchToState:178(630ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.381741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518709215983013:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:34.473582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518707736362876:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44905
2025-11-11T17:19:34.386948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096190:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.609752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518708386657729:2074] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.609914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518704091690401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.563987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518707627880744:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36128
2025-11-11T17:19:34.575706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983061:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:34.510793Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518711264096189:2067] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.474274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518707736362877:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44905
2025-11-11T17:19:34.585820Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518707627880744:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.606817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518709215983007:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(227884797ns) {Configure:104(227880272ns) {ProcessConfigured:135(64789ns) {StartInitialHandshake:157(62623ns) {DropHandshakes:469(2430ns) {DropIncomingHandshake:441(311ns) DropOutgoingHandshake:456(871ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:476(877ns) {SwitchToState:178(223ns)}}}}}
2025-11-11T17:19:34.625630Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518708386657717:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-11T17:19:34.625671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518703389191330:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(329721861ns) {Configure:104(329708977ns) {ProcessConfigured:135(52589980ns) {StartInitialHandshake:157(52586424ns) {DropHandshakes:469(3214ns) {DropIncomingHandshake:441(975ns) DropOutgoingHandshake:456(577ns)} GenerateSessionVirtualId:988(650ns) PrepareNewSessionHandshake:476(3992ns) {SwitchToState:178(1503ns)}}}}}
2025-11-11T17:19:34.474333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518707736362878:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44905
2025-11-11T17:19:34.581766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518711264096190:2068] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.645750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518707627880737:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.635780Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518709215983061:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.635870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518704921015693:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.636964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983067:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:34.661655Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518709215983061:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:34.676705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518704921015693:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.676752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:34.583382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518706969128871:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(141217348ns) {Configure:104(141212586ns) {ProcessConfigured:135(57105ns) {StartInitialHandshake:157(54991ns) {DropHandshakes:469(6222ns) {DropIncomingHandshake:441(714ns) DropOutgoingHandshake:456(565ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:476(2147ns) {SwitchToState:178(670ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.662819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518707684158699:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54303
2025-11-11T17:19:34.625684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518708386657717:2065] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.549653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362876:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.698020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518704921015693:2048] [node 1] ICP22 created new session: [5:7571518709215983069:2048]
2025-11-11T17:19:34.645805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518707627880737:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.583675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518711264096182:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.629801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518708386657724:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.672511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518707684158653:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(175274609ns) {Configure:104(175266196ns) {ProcessConfigured:135(70087ns) {StartInitialHandshake:157(67685ns) {DropHandshakes:469(3108ns) {DropIncomingHandshake:441(698ns) DropOutgoingHandshake:456(695ns)} GenerateSessionVirtualId:988(1328ns) PrepareNewSessionHandshake:476(1961ns) {SwitchToState:178(678ns)}}}}}
2025-11-11T17:19:34.583949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518706969128869:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.733688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518709215983069:2048] [node 1] ICS09 handshake done sender: [5:7571518709215983061:2113] self: [5:7571518709215983066:0] peer: [1:7571518707736362879:0] socket: 325 qp: -1
2025-11-11T17:19:34.662505Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518707627880737:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:34.574956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518707736362871:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(88996887ns) {Configure:104(88989821ns) {ProcessConfigured:135(63045ns) {StartInitialHandshake:157(60958ns) {DropHandshakes:469(4118ns) {DropIncomingHandshake:441(853ns) DropOutgoingHandshake:456(1164ns)} GenerateSessionVirtualId:988(2119ns) PrepareNewSessionHandshake:476(1761ns) {SwitchToState:178(675ns)}}}}}
2025-11-11T17:19:34.687436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518707684158644:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(465566180ns) {Configure:104(465562053ns) {ProcessConfigured:135(28598543ns) {StartInitialHandshake:157(28595839ns) {DropHandshakes:469(3193ns) {DropIncomingHandshake:441(635ns) DropOutgoingHandshake:456(679ns)} GenerateSessionVirtualId:988(479ns) PrepareNewSessionHandshake:476(2310ns) {SwitchToState:178(1357ns)}}}}}
2025-11-11T17:19:34.634067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518704091690402:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(264104351ns) {Configure:104(264095816ns) {ProcessConfigured:135(66244ns) {StartInitialHandshake:157(63277ns) {DropHandshakes:469(4812ns) {DropIncomingHandshake:441(539ns) DropOutgoingHandshake:456(237ns)} GenerateSessionVirtualId:988(685ns) PrepareNewSessionHandshake:476(1445ns) {SwitchToState:178(689ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.735879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518709215983069:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:34.705856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518707627880747:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36128
2025-11-11T17:19:34.590918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518711264096183:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(211859246ns) {Configure:104(211854921ns) {ProcessConfigured:135(46652ns) {StartInitialHandshake:157(44474ns) {DropHandshakes:469(1868ns) {DropIncomingHandshake:441(283ns) DropOutgoingHandshake:456(506ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:476(600ns) {SwitchToState:178(216ns)}}}}}
2025-11-11T17:19:34.729974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657779:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.617644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362878:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.710815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518707684158652:2069] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.749702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518704921015695:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(397336064ns) {Configure:104(397330497ns) {ProcessConfigured:135(24964624ns) {StartInitialHandshake:157(24961501ns) {DropHandshakes:469(1698ns) {DropIncomingHandshake:441(458ns) DropOutgoingHandshake:456(319ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:476(2864ns) {SwitchToState:178(1424ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.665665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096238:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.625630Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362862:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.713726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518707627880723:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(339741980ns) {Configure:104(339733790ns) {ProcessConfigured:135(31976688ns) {StartInitialHandshake:157(31972922ns) {DropHandshakes:469(2691ns) {DropIncomingHandshake:441(483ns) DropOutgoingHandshake:456(456ns)} GenerateSessionVirtualId:988(792ns) PrepareNewSessionHandshake:476(3736ns) {SwitchToState:178(1550ns)}}}}}
2025-11-11T17:19:34.734731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518708386657720:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(460466693ns) {Configure:104(460461846ns) {ProcessConfigured:135(24758860ns) {StartInitialHandshake:157(24756130ns) {DropHandshakes:469(9221343ns) {DropIncomingHandshake:441(1480ns) DropOutgoingHandshake:456(1290ns)} GenerateSessionVirtualId:988(1436ns) PrepareNewSessionHandshake:476(2131ns) {SwitchToState:178(371ns)}}}}}
2025-11-11T17:19:34.725935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518703389191331:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(507523318ns) {Configure:104(507517344ns) {ProcessConfigured:135(12029340ns) {StartInitialHandshake:157(12026550ns) {DropHandshakes:469(3432ns) {DropIncomingHandshake:441(773ns) DropOutgoingHandshake:456(562ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:476(3851ns) {SwitchToState:178(1697ns)}}}}}
2025-11-11T17:19:34.761802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.765782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518707627880747:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.766285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518704921015694:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(480355404ns) {Configure:104(480351769ns) {ProcessConfigured:135(8390739ns) {StartInitialHandshake:157(8387704ns) {DropHandshakes:469(1091ns) {DropIncomingHandshake:441(252ns) DropOutgoingHandshake:456(216ns)} GenerateSessionVirtualId:988(890ns) PrepareNewSessionHandshake:476(2371ns) {SwitchToState:178(632ns)}}}}}
2025-11-11T17:19:34.770388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518704921015695:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.785377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983075:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:34.805698Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518709215983063:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.733843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518703389191331:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.625671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518707736362862:2106] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.689704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096246:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.741785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518708386657779:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.770492Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518707627880734:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.709662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096247:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.742130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518704091690402:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.633737Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362877:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.734103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518707684158707:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54303
2025-11-11T17:19:34.805735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518709215983063:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.721629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.770519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518707627880734:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.773724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518707627880718:2099] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.777728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:34.793698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518707627880718:2099] [node 4] ICP22 created new session: [2:7571518707627880752:2099]
2025-11-11T17:19:34.795107Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518707627880742:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.795129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518707627880742:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.797680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518707627880752:2099] [node 4] ICS09 handshake done sender: [2:7571518707627880737:2110] self: [2:7571518707627880725:0] peer: [4:7571518708386657773:0] socket: 310 qp: -1
2025-11-11T17:19:34.813727Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518707627880734:2108] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:34.815469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518703332913374:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.815537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:34.815587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518703332913374:2049] [node 3] ICP22 created new session: [2:7571518707627880756:2049]
2025-11-11T17:19:34.815628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518707627880756:2049] [node 3] ICS09 handshake done sender: [2:7571518707627880734:2108] self: [2:7571518707627880728:0] peer: [3:7571518707684158708:0] socket: 305 qp: -1
2025-11-11T17:19:34.815668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518707627880756:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:34.815697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:34.821648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518707627880752:2099] [node 4] ICS10 traffic start
2025-11-11T17:19:34.826385Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518707627880748:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.826407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518707627880748:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.827321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518707627880760:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36128
2025-11-11T17:19:34.847292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:34.849744Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518707627880742:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:34.857649Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518707627880748:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:34.869693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518703332913373:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(523365079ns) {Configure:104(523359144ns) {ProcessConfigured:135(51893499ns) {StartInitialHandshake:157(51890300ns) {DropHandshakes:469(3053ns) {DropIncomingHandshake:441(795ns) DropOutgoingHandshake:456(724ns)} GenerateSessionVirtualId:988(1019ns) PrepareNewSessionHandshake:476(3505ns) {SwitchToState:178(976ns)}}}}}
2025-11-11T17:19:34.869751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518707627880760:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.873662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518707627880724:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.877728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:34.877775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518707627880724:2092] [node 6] ICP22 created new session: [2:7571518707627880763:2092]
2025-11-11T17:19:34.889904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518703332913373:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.894164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518707627880723:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.894207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:34.894251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518707627880723:2091] [node 5] ICP22 created new session: [2:7571518707627880764:2091]
2025-11-11T17:19:34.913682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518707627880763:2092] [node 6] ICS09 handshake done sender: [2:7571518707627880742:2112] self: [2:7571518707627880741:0] peer: [6:7571518711264096254:0] socket: 321 qp: -1
2025-11-11T17:19:34.914528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518707627880764:2091] [node 5] ICS09 handshake done sender: [2:7571518707627880748:2115] self: [2:7571518707627880746:0] peer: [5:7571518709215983078:0] socket: 353 qp: -1
2025-11-11T17:19:34.929689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518707627880763:2092] [node 6] ICS10 traffic start
2025-11-11T17:19:34.930955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518707627880764:2091] [node 5] ICS10 traffic start
2025-11-11T17:19:34.931048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:34.931502Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518707627880723:2091] [node 5] ICP35 event processing took too much time PendingConnection:278(37130917ns) {HandleHandshakeStatus:329(37129014ns) {DropIncomingHandshake:441(481ns) DropOutgoingHandshake:456(528ns) SwitchToState:178(457ns) ProcessPendingSessionEvents:495(100435ns) {ForwardSessionEventToSession:612(53502ns) ForwardSessionEventToSession:612(3744ns) ForwardSessionEventToSession:612(6478ns) ForwardSessionEventToSession:612(4352ns) ForwardSessionEventToSession:612(1998ns) ForwardSessionEventToSession:612(7749ns) ForwardSessionEventToSession:612(2166ns) ForwardSessionEventToSession:612(2870ns) ForwardSessionEventToSession:612(4812ns) ForwardSessionEventToSession:612(2370ns)}}}
2025-11-11T17:19:34.931612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518707627880723:2091] [node 5] ICP18 handshake (actor [2:7571518707627880760:2117]) is too old
2025-11-11T17:19:34.937815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.742209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657782:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.809086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983076:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:34.726312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518706969128870:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(444443542ns) {Configure:104(444439624ns) {ProcessConfigured:135(12493766ns) {StartInitialHandshake:157(12492231ns) {DropHandshakes:469(1800ns) {DropIncomingHandshake:441(422ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:476(3016ns) {SwitchToState:178(1923ns)}}}}}
2025-11-11T17:19:34.784034Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518711264096246:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.784141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518711264096248:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.784374Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518711264096238:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.784613Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518711264096190:2068] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:34.784826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518711264096183:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.784887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518706969128871:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.784927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518706969128870:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.785004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096251:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.793982Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:34.801685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518706969128869:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518711264096190:2068] held: no
2025-11-11T17:19:34.807077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518706969128869:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.807144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:34.807186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518706969128869:2048] [node 1] ICP22 created new session: [6:7571518711264096255:2048]
2025-11-11T17:19:34.807230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518711264096255:2048] [node 1] ICS09 handshake done sender: [6:7571518711264096190:2068] self: [6:7571518711264096242:0] peer: [1:7571518707736362861:0] socket: 315 qp: -1
2025-11-11T17:19:34.807262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518711264096255:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:34.807304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:34.808110Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518711264096189:2067] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:34.809446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518711264096182:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.809508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:34.809557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518711264096182:2063] [node 4] ICP22 created new session: [6:7571518711264096258:2063]
2025-11-11T17:19:34.809620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518711264096258:2063] [node 4] ICS09 handshake done sender: [6:7571518711264096189:2067] self: [6:7571518711264096241:0] peer: [4:7571518708386657716:0] socket: 307 qp: -1
2025-11-11T17:19:34.809655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518711264096258:2063] [node 4] ICS10 traffic start
2025-11-11T17:19:34.809696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:34.822286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096260:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.862580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096261:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.876554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518711264096263:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-11T17:19:34.882083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518711264096182:2063] [node 4] ICP35 event processing took too much time PendingConnection:278(72379542ns) {HandleHandshakeStatus:329(72377320ns) {DropIncomingHandshake:441(609ns) DropOutgoingHandshake:456(4736ns) SwitchToState:178(355ns) ProcessPendingSessionEvents:495(15600323ns) {ForwardSessionEventToSession:612(15537363ns) ForwardSessionEventToSession:612(12000ns) ForwardSessionEventToSession:612(8599ns) ForwardSessionEventToSession:612(3088ns) ForwardSessionEventToSession:612(5391ns) ForwardSessionEventToSession:612(2693ns) ForwardSessionEventToSession:612(6075ns) ForwardSessionEventToSession:612(2472ns) ForwardSessionEventToSession:612(3057ns) ForwardSessionEventToSession:612(8250ns)}}}
2025-11-11T17:19:34.882300Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8148) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:34.897830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518706969128871:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8148) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518711264096248:2119] held: no
2025-11-11T17:19:34.898173Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518711264096238:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:34.898638Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518711264096248:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:34.906500Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518711264096246:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:34.907895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518706969128871:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.907930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:34.907971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518706969128871:2050] [node 3] ICP22 created new session: [6:7571518711264096264:2050]
2025-11-11T17:19:34.908019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518711264096264:2050] [node 3] ICS09 handshake done sender: [6:7571518711264096248:2119] self: [6:7571518711264096253:0] peer: [3:7571518707684158700:0] socket: 339 qp: -1
2025-11-11T17:19:34.908049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518711264096264:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:34.908077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:34.908375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518706969128870:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.908437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:34.908479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518706969128870:2049] [node 2] ICP22 created new session: [6:7571518711264096266:2049]
2025-11-11T17:19:34.908515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518711264096266:2049] [node 2] ICS09 handshake done sender: [6:7571518711264096238:2114] self: [6:7571518711264096254:0] peer: [2:7571518707627880741:0] socket: 322 qp: -1
2025-11-11T17:19:34.908538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518711264096266:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:34.908560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:34.925896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518711264096183:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.925947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:34.925993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518711264096183:2062] [node 5] ICP22 created new session: [6:7571518711264096268:2062]
2025-11-11T17:19:34.926036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518711264096268:2062] [node 5] ICS09 handshake done sender: [6:7571518711264096246:2117] self: [6:7571518711264096252:0] peer: [5:7571518709215983062:0] socket: 328 qp: -1
2025-11-11T17:19:34.926065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518711264096268:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:34.926097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:34.636106Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362880:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.737658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518707684158699:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.771179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518708386657782:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.974076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518707627880724:2092] [node 6] ICP35 event processing took too much time PendingConnection:278(104100723ns) {HandleHandshakeStatus:329(104098257ns) {DropIncomingHandshake:441(920ns) DropOutgoingHandshake:456(2682ns) SwitchToState:178(1062ns) ProcessPendingSessionEvents:495(120017ns) {ForwardSessionEventToSession:612(65120ns) ForwardSessionEventToSession:612(5458ns) ForwardSessionEventToSession:612(8349ns) ForwardSessionEventToSession:612(3582ns) ForwardSessionEventToSession:612(2494ns) ForwardSessionEventToSession:612(8472ns) ForwardSessionEventToSession:612(2488ns) ForwardSessionEventToSession:612(4911ns) ForwardSessionEventToSession:612(1892ns) ForwardSessionEventToSession:612(5331ns)}}}
2025-11-11T17:19:34.636123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518707736362880:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.809141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983077:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:34.822469Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518709215983076:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.822571Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518709215983009:2066] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:34.822723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518704921015694:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.823137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518704921015695:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.823186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:34.823230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518704921015695:2050] [node 3] ICP22 created new session: [5:7571518709215983079:2050]
2025-11-11T17:19:34.823273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518709215983079:2050] [node 3] ICS09 handshake done sender: [5:7571518709215983009:2066] self: [5:7571518709215983074:0] peer: [3:7571518707684158646:0] socket: 303 qp: -1
2025-11-11T17:19:34.823307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518709215983079:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:34.823341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:34.846456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983082:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:34.853808Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518709215983076:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:34.863321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518709215983063:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:34.869825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518704921015694:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.869884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:34.871734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518704921015694:2049] [node 2] ICP22 created new session: [5:7571518709215983084:2049]
2025-11-11T17:19:34.893868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518709215983084:2049] [node 2] ICS09 handshake done sender: [5:7571518709215983076:2119] self: [5:7571518709215983078:0] peer: [2:7571518707627880746:0] socket: 354 qp: -1
2025-11-11T17:19:34.893913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518709215983084:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:34.893958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:34.905696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518709215983077:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.906764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518709215983007:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.906805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:34.906848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518709215983007:2063] [node 6] ICP22 created new session: [5:7571518709215983086:2063]
2025-11-11T17:19:34.906888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518709215983086:2063] [node 6] ICS09 handshake done sender: [5:7571518709215983063:2114] self: [5:7571518709215983062:0] peer: [6:7571518711264096252:0] socket: 327 qp: -1
2025-11-11T17:19:34.906922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518709215983086:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:34.906959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:34.907456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518709215983007:2063] [node 6] ICP18 handshake (actor [5:7571518709215983077:2120]) is too old
2025-11-11T17:19:34.937719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518704921015693:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(192953558ns) {HandleHandshakeStatus:329(192951600ns) {DropIncomingHandshake:441(458ns) DropOutgoingHandshake:456(4224ns) SwitchToState:178(697ns) ProcessPendingSessionEvents:495(75956247ns) {ForwardSessionEventToSession:612(15969189ns) ForwardSessionEventToSession:612(416004ns) ForwardSessionEventToSession:612(4496ns) ForwardSessionEventToSession:612(8903314ns) ForwardSessionEventToSession:612(8506510ns) ForwardSessionEventToSession:612(4500ns) ForwardSessionEventToSession:612(3024ns) ForwardSessionEventToSession:612(22135361ns) ForwardSessionEventToSession:612(3971ns) ForwardSessionEventToSession:612(19986785ns)}}}
2025-11-11T17:19:34.970144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518709215983088:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:34.983353Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518709215983088:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.989945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518709215983013:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.781901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518707684158703:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.781940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518707684158703:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.782541Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518707684158647:2066] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:35.005691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518704921015694:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(57791742ns) {HandleHandshakeStatus:329(57789224ns) {DropIncomingHandshake:441(816ns) DropOutgoingHandshake:456(5928ns) SwitchToState:178(639ns) ProcessPendingSessionEvents:495(33524400ns) {ForwardSessionEventToSession:612(15596930ns) ForwardSessionEventToSession:612(7883ns) ForwardSessionEventToSession:612(3118ns) ForwardSessionEventToSession:612(2323ns) ForwardSessionEventToSession:612(4907ns) ForwardSessionEventToSession:612(8627ns) ForwardSessionEventToSession:612(2000835ns) ForwardSessionEventToSession:612(4007ns) ForwardSessionEventToSession:612(2504ns) ForwardSessionEventToSession:612(15879411ns)}}}
2025-11-11T17:19:34.986165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518707627880765:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36128
2025-11-11T17:19:34.986342Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:34.993679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518707627880724:2092] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:34.998976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518707627880718:2099] [node 4] ICP35 event processing took too much time PendingConnection:278(137321773ns) {HandleHandshakeStatus:329(137319074ns) {DropIncomingHandshake:441(3936646ns) DropOutgoingHandshake:456(1170ns) SwitchToState:178(576ns) ProcessPendingSessionEvents:495(56414096ns) {ForwardSessionEventToSession:612(11521793ns) ForwardSessionEventToSession:612(5304030ns) ForwardSessionEventToSession:612(5038ns) ForwardSessionEventToSession:612(3565ns) ForwardSessionEventToSession:612(5441595ns) ForwardSessionEventToSession:612(4441ns) ForwardSessionEventToSession:612(12935298ns) ForwardSessionEventToSession:612(12042155ns) ForwardSessionEventToSession:612(3032ns) ForwardSessionEventToSession:612(9127017ns)}}}
2025-11-11T17:19:35.005768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518707627880766:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36128
2025-11-11T17:19:34.773828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518708386657717:2065] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:34.782560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518707684158647:2066] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.637158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518707736362880:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:34.652936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518703441395511:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(390820959ns) {Configure:104(390812855ns) {ProcessConfigured:135(77359ns) {StartInitialHandshake:157(74421ns) {DropHandshakes:469(3669ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(985ns)} GenerateSessionVirtualId:988(888ns) PrepareNewSessionHandshake:476(1950ns) {SwitchToState:178(689ns)}}}}}
2025-11-11T17:19:34.673675Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518707736362862:2106] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:34.674891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518707736362875:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(127525963ns) {Configure:104(127520352ns) {ProcessConfigured:135(48625ns) {StartInitialHandshake:157(46701ns) {DropHandshakes:469(2648ns) {DropIncomingHandshake:441(595ns) DropOutgoingHandshake:456(368ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:476(1627ns) {SwitchToState:178(695ns)}}}}}
2025-11-11T17:19:34.675030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518707736362871:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.675080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:34.675144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518707736362871:2092] [node 5] ICP22 created new session: [1:7571518707736362893:2092]
2025-11-11T17:19:34.675184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518707736362893:2092] [node 5] ICS09 handshake done sender: [1:7571518707736362880:2114] self: [1:7571518707736362879:0] peer: [5:7571518709215983066:0] socket: 324 qp: -1
2025-11-11T17:19:34.675395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518707736362893:2092] [node 5] ICS10 traffic start
2025-11-11T17:19:34.675430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:34.733657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518703441395512:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(491487399ns) {Configure:104(491482572ns) {ProcessConfigured:135(51674035ns) {StartInitialHandshake:157(51672126ns) {DropHandshakes:469(2694ns) {DropIncomingHandshake:441(429ns) DropOutgoingHandshake:456(507ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:476(2167ns) {SwitchToState:178(118ns)}}}}}
2025-11-11T17:19:34.749675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518707736362860:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.753854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:34.753907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518707736362860:2093] [node 6] ICP22 created new session: [1:7571518707736362898:2093]
2025-11-11T17:19:34.781860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518707736362898:2093] [node 6] ICS09 handshake done sender: [1:7571518707736362862:2106] self: [1:7571518707736362861:0] peer: [6:7571518711264096242:0] socket: 301 qp: -1
2025-11-11T17:19:34.785751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362892:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.785773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518707736362892:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.793794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518707736362898:2093] [node 6] ICS10 traffic start
2025-11-11T17:19:34.806046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:34.806540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518707736362860:2093] [node 6] ICP35 event processing took too much time PendingConnection:278(68621704ns) {HandleHandshakeStatus:329(68619857ns) {DropIncomingHandshake:441(18880ns) DropOutgoingHandshake:456(567ns) SwitchToState:178(1203ns) ProcessPendingSessionEvents:495(117405ns) {ForwardSessionEventToSession:612(68693ns) ForwardSessionEventToSession:612(3447ns) ForwardSessionEventToSession:612(7993ns) ForwardSessionEventToSession:612(5141ns) ForwardSessionEventToSession:612(2310ns) ForwardSessionEventToSession:612(8207ns) ForwardSessionEventToSession:612(2859ns) ForwardSessionEventToSession:612(2541ns) ForwardSessionEventToSession:612(4949ns) ForwardSessionEventToSession:612(2702ns)}}}
2025-11-11T17:19:34.810027Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518707736362892:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:34.850170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518707736362875:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.850227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:34.850282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518707736362875:2094] [node 4] ICP22 created new session: [1:7571518707736362902:2094]
2025-11-11T17:19:34.850320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518707736362902:2094] [node 4] ICS09 handshake done sender: [1:7571518707736362892:2117] self: [1:7571518707736362891:0] peer: [4:7571518708386657785:0] socket: 304 qp: -1
2025-11-11T17:19:34.850359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518707736362902:2094] [node 4] ICS10 traffic start
2025-11-11T17:19:34.850394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:34.857754Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362890:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.857790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518707736362890:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.869736Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518707736362890:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:34.883121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518703441395512:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.883165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:34.883212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518703441395512:2049] [node 3] ICP22 created new session: [1:7571518707736362905:2049]
2025-11-11T17:19:34.883252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518707736362905:2049] [node 3] ICS09 handshake done sender: [1:7571518707736362890:2116] self: [1:7571518707736362883:0] peer: [3:7571518707684158714:0] socket: 340 qp: -1
2025-11-11T17:19:34.883291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518707736362905:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:34.883321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:34.888296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518707736362907:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44905
2025-11-11T17:19:34.921776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362887:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.921809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518707736362887:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.922741Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518707736362887:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:34.922990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518703441395511:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.923029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:34.923066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518703441395511:2048] [node 2] ICP22 created new session: [1:7571518707736362909:2048]
2025-11-11T17:19:34.923107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518707736362909:2048] [node 2] ICS09 handshake done sender: [1:7571518707736362887:2115] self: [1:7571518707736362886:0] peer: [2:7571518707627880762:0] socket: 332 qp: -1
2025-11-11T17:19:34.923145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518707736362909:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:34.923175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:34.981950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518707736362901:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44905
2025-11-11T17:19:34.993680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362901:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:35.001704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518703441395511:2048] [node 2] ICP18 handshake (actor [1:7571518707736362901:2118]) is too old
2025-11-11T17:19:34.786423Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518707684158703:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:35.047192Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518707736362907:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:35.047337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518703441395512:2049] [node 3] ICP18 handshake (actor [1:7571518707736362907:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:34.796484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518704091690400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.790241Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518707684158647:2066] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:34.796204Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518707684158707:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.821647Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518707684158701:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:34.821689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518707684158701:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:34.830555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518703389191330:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:34.830631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518707684158710:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54303
2025-11-11T17:19:34.830793Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:34.833913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518707684158645:2064] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:34.834018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518707684158713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54303
2025-11-11T17:19:34.862776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518707684158701:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:34.864893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518707684158653:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.864950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:34.864991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518707684158653:2067] [node 6] ICP22 created new session: [3:7571518707684158717:2067]
2025-11-11T17:19:34.865035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518707684158717:2067] [node 6] ICS09 handshake done sender: [3:7571518707684158701:2114] self: [3:7571518707684158700:0] peer: [6:7571518711264096253:0] socket: 338 qp: -1
2025-11-11T17:19:34.865070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518707684158717:2067] [node 6] ICS10 traffic start
2025-11-11T17:19:34.865103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:34.865674Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518707684158652:2069] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:34.885888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518707684158644:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.885922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:34.885962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518707684158644:2065] [node 4] ICP22 created new session: [3:7571518707684158719:2065]
2025-11-11T17:19:34.886009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518707684158719:2065] [node 4] ICS09 handshake done sender: [3:7571518707684158703:2115] self: [3:7571518707684158698:0] peer: [4:7571518708386657783:0] socket: 341 qp: -1
2025-11-11T17:19:34.886044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518707684158719:2065] [node 4] ICS10 traffic start
2025-11-11T17:19:34.886072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:34.886411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518707684158645:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:34.886435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:34.886470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518707684158645:2064] [node 5] ICP22 created new session: [3:7571518707684158721:2064]
2025-11-11T17:19:34.886502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518707684158721:2064] [node 5] ICS09 handshake done sender: [3:7571518707684158647:2066] self: [3:7571518707684158646:0] peer: [5:7571518709215983074:0] socket: 302 qp: -1
2025-11-11T17:19:34.886524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518707684158721:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:34.886551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:34.886873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518703389191331:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.886931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:34.886974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518703389191331:2049] [node 2] ICP22 created new session: [3:7571518707684158723:2049]
2025-11-11T17:19:34.887012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518707684158723:2049] [node 2] ICS09 handshake done sender: [3:7571518707684158652:2069] self: [3:7571518707684158708:0] peer: [2:7571518707627880728:0] socket: 306 qp: -1
2025-11-11T17:19:34.887042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518707684158723:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:34.887065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:34.900323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518707684158727:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54303
2025-11-11T17:19:34.935262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518707684158716:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54303
2025-11-11T17:19:34.942230Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:34.985652Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518707684158707:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:34.990691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518703389191330:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:35.021895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:35.034748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518703389191330:2048] [node 1] ICP22 created new session: [3:7571518711979126024:2048]
2025-11-11T17:19:35.053712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518711979126024:2048] [node 1] ICS09 handshake done sender: [3:7571518707684158707:2117] self: [3:7571518707684158714:0] peer: [1:7571518707736362883:0] socket: 342 qp: -1
2025-11-11T17:19:35.053762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518711979126024:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:35.050340Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518707627880747:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:34.818137Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8333) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:34.819038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518708386657715:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:35.050111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518713510950386:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27189
2025-11-11T17:19:35.053649Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:34.829688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518704091690400:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19955654ns) {IncomingHandshake:261(19952760ns) {DropIncomingHandshake:441(518ns) PrepareNewSessionHandshake:476(2348ns) {SwitchToState:178(823ns)} GenerateSessionVirtualId:988(707ns) IssueIncomingHandshakeReply:193(14649774ns) {SwitchToState:178(2160ns)}}}
2025-11-11T17:19:34.829776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518704091690401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8333) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518708386657729:2074] held: no
2025-11-11T17:19:34.863579Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:34.873702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:34.873757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518708386657715:2064] [node 6] ICP22 created new session: [4:7571518708386657787:2064]
2025-11-11T17:19:34.896496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518704091690400:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518708386657782:2120] held: no
2025-11-11T17:19:34.896927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657786:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
2025-11-11T17:19:34.906203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657788:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
2025-11-11T17:19:34.910175Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518708386657779:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:34.910381Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518708386657782:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:34.910597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518704091690402:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.910634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:34.910673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518704091690402:2050] [node 3] ICP22 created new session: [4:7571518708386657791:2050]
2025-11-11T17:19:34.916460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518708386657791:2050] [node 3] ICS09 handshake done sender: [4:7571518708386657779:2118] self: [4:7571518708386657783:0] peer: [3:7571518707684158698:0] socket: 314 qp: -1
2025-11-11T17:19:34.916503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518708386657791:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:34.916552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:34.917163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518704091690400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:34.917228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:34.917271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518704091690400:2048] [node 1] ICP22 created new session: [4:7571518708386657793:2048]
2025-11-11T17:19:34.917305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518708386657793:2048] [node 1] ICS09 handshake done sender: [4:7571518708386657782:2120] self: [4:7571518708386657785:0] peer: [1:7571518707736362891:0] socket: 343 qp: -1
2025-11-11T17:19:34.917330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518708386657793:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:34.917376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:34.935198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518704091690401:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(39918030ns) {HandleHandshakeStatus:395(39915975ns) {LogHandshakeFail:472(12851ns) {UpdateErrorStateLog:544(4868ns)} DropOutgoingHandshake:456(613ns) ProcessPendingSessionEvents:495(39861181ns) {DropSessionEvent:512(26885ns) DropSessionEvent:512(5445ns) DropSessionEvent:512(7028081ns) DropSessionEvent:512(4773ns) DropSessionEvent:512(3868ns) DropSessionEvent:512(4253ns) DropSessionEvent:512(4452ns) DropSessionEvent:512(4247ns) DropSessionEvent:512(23035ns) DropSessionEvent:512(4539ns)}}}
2025-11-11T17:19:34.941899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518708386657787:2064] [node 6] ICS09 handshake done sender: [4:7571518708386657717:2065] self: [4:7571518708386657716:0] peer: [6:7571518711264096241:0] socket: 300 qp: -1
2025-11-11T17:19:34.945670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518708386657777:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27713
2025-11-11T17:19:34.953769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518708386657787:2064] [node 6] ICS10 traffic start
2025-11-11T17:19:34.953838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:34.982761Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518708386657729:2074] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:35.001869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518704091690401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:35.010023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:35.021727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518708386657780:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:35.021779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518708386657780:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:35.021991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518704091690401:2049] [node 2] ICP22 created new session: [4:7571518712681625092:2049]
2025-11-11T17:19:35.034809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518712681625092:2049] [node 2] ICS09 handshake done sender: [4:7571518708386657729:2074] self: [4:7571518708386657773:0] peer: [2:7571518707627880725:0] socket: 311 qp: -1
2025-11-11T17:19:35.041834Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518708386657780:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:35.045728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518712681625092:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:35.053853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:35.054168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518704091690401:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(52231500ns) {HandleHandshakeStatus:329(52228644ns) {DropIncomingHandshake:441(857ns) DropOutgoingHandshake:456(724ns) SwitchToState:178(632ns) ProcessPendingSessionEvents:495(1212ns)}}
2025-11-11T17:19:35.093865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518708386657720:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:35.093935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:35.093983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518708386657720:2066] [node 5] ICP22 created new session: [4:7571518712681625095:2066]
2025-11-11T17:19:35.094034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518712681625095:2066] [node 5] ICS09 handshake done sender: [4:7571518708386657780:2119] self: [4:7571518708386657778:0] peer: [5:7571518709215983089:0] socket: 309 qp: -1
2025-11-11T17:19:35.094078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518712681625095:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:35.094114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:35.069772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:35.074079Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518709215983088:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:35.121720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518708386657715:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(247757982ns) {HandleHandshakeStatus:329(247755371ns) {DropIncomingHandshake:441(15146ns) DropOutgoingHandshake:456(488ns) SwitchToState:178(350ns) ProcessPendingSessionEvents:495(43880985ns) {ForwardSessionEventToSession:612(8225247ns) ForwardSessionEventToSession:612(7713952ns) ForwardSessionEventToSession:612(15835ns) ForwardSessionEventToSession:612(4266531ns) ForwardSessionEventToSession:612(4436ns) ForwardSessionEventToSession:612(2365ns) ForwardSessionEventToSession:612(4343795ns) ForwardSessionEventToSession:612(3356ns) ForwardSessionEventToSession:612(11339911ns) ForwardSessionEventToSession:612(7930247ns)}}}
2025-11-11T17:19:35.074337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518709215983013:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:35.074415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:35.074472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518709215983013:2064] [node 4] ICP22 created new session: [5:7571518713510950387:2064]
2025-11-11T17:19:35.074514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518713510950387:2064] [node 4] ICS09 handshake done sender: [5:7571518709215983088:2122] self: [5:7571518709215983089:0] peer: [4:7571518708386657778:0] socket: 356 qp: -1
2025-11-11T17:19:35.074555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518713510950387:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:35.074596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:35.057920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518703332913373:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518707627880747:2114] held: no
2025-11-11T17:19:35.085915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518703332913373:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:35.085967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:35.086010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518703332913373:2048] [node 1] ICP22 created new session: [2:7571518711922848065:2048]
2025-11-11T17:19:35.086052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518711922848065:2048] [node 1] ICS09 handshake done sender: [2:7571518707627880747:2114] self: [2:7571518707627880762:0] peer: [1:7571518707736362886:0] socket: 333 qp: -1
2025-11-11T17:19:35.086087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518711922848065:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:35.086133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:35.134040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518703389191330:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(143073531ns) {HandleHandshakeStatus:329(143070863ns) {DropIncomingHandshake:441(644ns) DropOutgoingHandshake:456(4634ns) SwitchToState:178(121ns) ProcessPendingSessionEvents:495(23989084ns) {ForwardSessionEventToSession:612(23933449ns) ForwardSessionEventToSession:612(7455ns) ForwardSessionEventToSession:612(3238ns) ForwardSessionEventToSession:612(6974ns) ForwardSessionEventToSession:612(1901ns) ForwardSessionEventToSession:612(5216ns) ForwardSessionEventToSession:612(2689ns) ForwardSessionEventToSession:612(5439ns) ForwardSessionEventToSession:612(9087ns) ForwardSessionEventToSession:612(1956ns)}}}
2025-11-11T17:19:35.235239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:35.235467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518711922848065:2048] [node 1] ICS01 socket: 333 reason# EndOfStream
2025-11-11T17:19:35.235506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518703332913373:2048] [node 1] ICP30 unregister session Session# [2:7571518711922848065:2048] VirtualId# [2:7571518707627880762:0]
2025-11-11T17:19:35.235535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518711922848065:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.235605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:35.235640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:35.239864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:35.240190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518711979126024:2048] [node 1] ICS07 socket disconnect 342 reason# EndOfStream
2025-11-11T17:19:35.240218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518711979126024:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.240323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:35.240369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518711979126024:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:35.245952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:35.246131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518709215983069:2048] [node 1] ICS07 socket disconnect 325 reason# EndOfStream
2025-11-11T17:19:35.246152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518709215983069:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.246210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:35.246223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:35.246427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518708386657793:2048] [node 1] ICS01 socket: 343 reason# EndOfStream
2025-11-11T17:19:35.246257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518709215983069:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:35.246457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518704091690400:2048] [node 1] ICP30 unregister session Session# [4:7571518708386657793:2048] VirtualId# [4:7571518708386657785:0]
2025-11-11T17:19:35.246484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518708386657793:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.246540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:35.246572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:35.245727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:35.245951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518711264096255:2048] [node 1] ICS01 socket: 315 reason# EndOfStream
2025-11-11T17:19:35.245982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518706969128869:2048] [node 1] ICP30 unregister session Session# [6:7571518711264096255:2048] VirtualId# [6:7571518711264096242:0]
2025-11-11T17:19:35.246014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518711264096255:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.246086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:35.246124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:35.261752Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address
2025-11-11T17:19:35.262116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518703389191330:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:35.262176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518711979126024:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:35.262213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518703389191330:2048] [node 1] ICP30 unregister session Session# [3:7571518711979126024:2048] VirtualId# [3:7571518707684158714:0]
2025-11-11T17:19:35.262272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:35.262302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518703389191330:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:35.262318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:35.247260Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address
2025-11-11T17:19:35.266635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518704921015693:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:35.274203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518709215983069:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:35.274247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518704921015693:2048] [node 1] ICP30 unregister session Session# [5:7571518709215983069:2048] VirtualId# [5:7571518709215983066:0]
2025-11-11T17:19:35.274298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:35.274335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518704921015693:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:35.274350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:35.274587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [5:7571518704921015693:2048] [node 1] ICP35 event processing took too much time StateWork:293(26910754ns) {HandleHandshakeStatus:395(26907957ns) {LogHandshakeFail:472(10227662ns) {UpdateErrorStateLog:544(5643ns)} DropOutgoingHandshake:456(4285ns) ProcessPendingSessionEvents:495(333ns) UnregisterSession:538(42777ns) {ProcessPendingSessionEvents:495(432ns) SwitchToInitialState:201(1371ns) {SwitchToState:178(663ns)}} TransitToErrorState:752(53676ns) {SwitchToState:178(5425ns) ProcessPendingSessionEvents:495(145ns) DropHandshakes:469(2746ns) {DropIncomingHandshake:441(538ns) DropOutgoingHandshake:456(461ns)}}}}
2025-11-11T17:19:35.302064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518704921015693:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:35.305751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518703389191330:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:35.357314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:35.357577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518707684158723:2049] [node 2] ICS01 socket: 306 reason# EndOfStream
2025-11-11T17:19:35.357630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518703389191331:2049] [node 2] ICP30 unregister session Session# [3:7571518707684158723:2049] VirtualId# [3:7571518707684158708:0]
2025-11-11T17:19:35.357661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518707684158723:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.357732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:35.357770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:35.381738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:35.381973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518711264096266:2049] [node 2] ICS01 socket: 322 reason# EndOfStream
2025-11-11T17:19:35.382012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518706969128870:2049] [node 2] ICP30 unregister session Session# [6:7571518711264096266:2049] VirtualId# [6:7571518711264096254:0]
2025-11-11T17:19:35.382044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518711264096266:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.382126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:35.382168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:35.382627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:35.381737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:35.382027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518709215983084:2049] [node 2] ICS07 socket disconnect 354 reason# EndOfStream
2025-11-11T17:19:35.382052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518709215983084:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.382126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:35.382169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518709215983084:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:35.383540Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8333) Couldn't connect to any resolved address
2025-11-11T17:19:35.383820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518704921015694:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8333) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:35.383861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518709215983084:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:35.383897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518704921015694:2049] [node 2] ICP30 unregister session Session# [5:7571518709215983084:2049] VirtualId# [5:7571518709215983078:0]
2025-11-11T17:19:35.383936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:35.383966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518704921015694:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8333) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:35.383985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8333) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:35.382893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518712681625092:2049] [node 2] ICS01 socket: 311 reason# EndOfStream
2025-11-11T17:19:35.382951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518704091690401:2049] [node 2] ICP30 unregister session Session# [4:7571518712681625092:2049] VirtualId# [4:7571518708386657773:0]
2025-11-11T17:19:35.382981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518712681625092:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.383031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:35.383071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:35.393564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518704921015694:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:35.578235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:35.578427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518709215983079:2050] [node 3] ICS01 socket: 303 reason# EndOfStream
2025-11-11T17:19:35.578467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518704921015695:2050] [node 3] ICP30 unregister session Session# [5:7571518709215983079:2050] VirtualId# [5:7571518709215983074:0]
2025-11-11T17:19:35.578494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518709215983079:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.578604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:35.578645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:35.577876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:35.578169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518708386657791:2050] [node 3] ICS01 socket: 314 reason# EndOfStream
2025-11-11T17:19:35.578209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518704091690402:2050] [node 3] ICP30 unregister session Session# [4:7571518708386657791:2050] VirtualId# [4:7571518708386657783:0]
2025-11-11T17:19:35.578251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518708386657791:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.578328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:35.578373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:35.578451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:35.578702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518711264096264:2050] [node 3] ICS01 socket: 339 reason# EndOfStream
2025-11-11T17:19:35.578738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518706969128871:2050] [node 3] ICP30 unregister session Session# [6:7571518711264096264:2050] VirtualId# [6:7571518711264096253:0]
2025-11-11T17:19:35.578761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518711264096264:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.578804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:35.578836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:35.645774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:35.646012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518711264096258:2063] [node 4] ICS01 socket: 307 reason# EndOfStream
2025-11-11T17:19:35.646051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518711264096182:2063] [node 4] ICP30 unregister session Session# [6:7571518711264096258:2063] VirtualId# [6:7571518711264096241:0]
2025-11-11T17:19:35.646082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518711264096258:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.646175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:35.646219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:35.646296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:35.646464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518713510950387:2064] [node 4] ICS01 socket: 356 reason# EndOfStream
2025-11-11T17:19:35.646499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518709215983013:2064] [node 4] ICP30 unregister session Session# [5:7571518713510950387:2064] VirtualId# [5:7571518709215983089:0]
2025-11-11T17:19:35.646520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518713510950387:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.646564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:35.646592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:35.729824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:35.730206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518711264096268:2062] [node 5] ICS01 socket: 328 reason# EndOfStream
2025-11-11T17:19:35.730298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518711264096183:2062] [node 5] ICP30 unregister session Session# [6:7571518711264096268:2062] VirtualId# [6:7571518711264096252:0]
2025-11-11T17:19:35.730362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518711264096268:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:35.730545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:35.730628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:36.081797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518716874623792:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:36.081845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518716874623793:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:36.161747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518718775863261:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:36.161815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518718775863262:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:36.229844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518719341979511:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:36.229892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518719341979512:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:36.265893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518717758140709:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:36.265938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518717758140710:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:36.265965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518717758140711:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:36.347113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518717353739408:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:36.347164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518717353739409:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:36.347190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518717353739410:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:36.415852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518715748609249:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:36.415908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518715748609250:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:36.415937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518715748609251:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:36.632534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518715748609262:2059] [node 5] ICP01 ready to work
2025-11-11T17:19:36.662120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518719341979541:2071] [node 6] ICP01 ready to work
2025-11-11T17:19:36.662446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518719341979542:2072] [node 5] ICP01 ready to work
2025-11-11T17:19:36.790052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518718775863316:2078] [node 5] ICP01 ready to work
2025-11-11T17:19:36.790455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518718775863317:2080] [node 4] ICP01 ready to work
2025-11-11T17:19:36.769967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518716874623856:2106] [node 6] ICP01 ready to work
2025-11-11T17:19:36.860934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518717758140710:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(226618087ns) {Configure:104(226614503ns) {ProcessConfigured:135(15054952ns) {StartInitialHandshake:157(15051753ns) {DropHandshakes:469(2924ns) {DropIncomingHandshake:441(632ns) DropOutgoingHandshake:456(587ns)} GenerateSessionVirtualId:988(750ns) PrepareNewSessionHandshake:476(3704ns) {SwitchToState:178(1636ns)}}}}}
2025-11-11T17:19:36.829732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518718775863323:2084] [node 6] ICP01 ready to work
2025-11-11T17:19:36.861459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518717758140775:2067] [node 5] ICP01 ready to work
2025-11-11T17:19:36.869956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518719341979511:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(192480496ns) {Configure:104(192477507ns) {ProcessConfigured:135(53942ns) {StartInitialHandshake:157(51161ns) {DropHandshakes:469(2971ns) {DropIncomingHandshake:441(562ns) DropOutgoingHandshake:456(753ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:476(1958ns) {SwitchToState:178(696ns)}}}}}
2025-11-11T17:19:36.871902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518719341979584:2075] [node 4] ICP01 ready to work
2025-11-11T17:19:36.913710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518716874623857:2107] [node 4] ICP01 ready to work
2025-11-11T17:19:36.925933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518715748609262:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(293088222ns) {Configure:104(293083276ns) {ProcessConfigured:135(4040115ns) {StartInitialHandshake:157(4036532ns) {DropHandshakes:469(2866ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(874ns) PrepareNewSessionHandshake:476(5172ns) {SwitchToState:178(2817ns)}}}}}
2025-11-11T17:19:36.861812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518717758140776:2071] [node 6] ICP01 ready to work
2025-11-11T17:19:36.926418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518715748609315:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:36.935027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518718775863317:2080] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(65204624ns) {Configure:104(65197605ns) {ProcessConfigured:135(45671ns) {StartInitialHandshake:157(43424ns) {DropHandshakes:469(1846ns) {DropIncomingHandshake:441(412ns) DropOutgoingHandshake:456(168ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:476(972ns) {SwitchToState:178(326ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:36.950177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518719341979591:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47448
2025-11-11T17:19:36.966173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518717353739409:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(288255358ns) {Configure:104(288246822ns) {ProcessConfigured:135(56973ns) {StartInitialHandshake:157(54346ns) {DropHandshakes:469(2761ns) {DropIncomingHandshake:441(536ns) DropOutgoingHandshake:456(472ns)} GenerateSessionVirtualId:988(860ns) PrepareNewSessionHandshake:476(1788ns) {SwitchToState:178(694ns)}}}}}
2025-11-11T17:19:36.966673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518717353739473:2066] [node 6] ICP01 ready to work
2025-11-11T17:19:36.966978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518717353739474:2068] [node 4] ICP01 ready to work
2025-11-11T17:19:36.994269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518717353739473:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25235569ns) {Configure:104(25232627ns) {ProcessConfigured:135(51836ns) {StartInitialHandshake:157(49432ns) {DropHandshakes:469(2571ns) {DropIncomingHandshake:441(556ns) DropOutgoingHandshake:456(684ns)} GenerateSessionVirtualId:988(863ns) PrepareNewSessionHandshake:476(1493ns) {SwitchToState:178(300ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:36.997182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518717353739475:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:36.959189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518718775863316:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21622159ns) {Configure:104(21619763ns) {ProcessConfigured:135(58837ns) {StartInitialHandshake:157(56137ns) {DropHandshakes:469(5275ns) {DropIncomingHandshake:441(548ns) DropOutgoingHandshake:456(364ns)} GenerateSessionVirtualId:988(723ns) PrepareNewSessionHandshake:476(1809ns) {SwitchToState:178(579ns)}}}}}
2025-11-11T17:19:36.997253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518717353739476:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
2025-11-11T17:19:36.985845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518716874623858:2108] [node 5] ICP01 ready to work
2025-11-11T17:19:37.025184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518717758140788:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24060
2025-11-11T17:19:37.046048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518715748609315:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(116554814ns) {Configure:104(116550687ns) {ProcessConfigured:135(68044ns) {StartInitialHandshake:157(65092ns) {DropHandshakes:469(8441ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(509ns)} GenerateSessionVirtualId:988(780ns) PrepareNewSessionHandshake:476(2358ns) {SwitchToState:178(880ns)}}}}}
2025-11-11T17:19:37.031712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518723636946888:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47448
2025-11-11T17:19:36.959877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518718775863337:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39515
2025-11-11T17:19:37.065864Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518718775863337:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.067496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518716874623793:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(78141359ns) {Configure:104(78137060ns) {ProcessConfigured:135(1375848ns) {StartInitialHandshake:157(1373482ns) {DropHandshakes:469(2881ns) {DropIncomingHandshake:441(724ns) DropOutgoingHandshake:456(629ns)} GenerateSessionVirtualId:988(696ns) PrepareNewSessionHandshake:476(2712ns) {SwitchToState:178(1029ns)}}}}}
2025-11-11T17:19:37.057769Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518717353739475:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.057926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518717353739474:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.067354Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518717353739476:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.067486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518717353739410:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.080748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518723070830637:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39515
2025-11-11T17:19:37.037756Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518719341979591:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.073960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518715748609249:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27679075ns) {Configure:104(27673784ns) {ProcessConfigured:135(6216546ns) {StartInitialHandshake:157(6213162ns) {DropHandshakes:469(1092ns) {DropIncomingHandshake:441(265ns) DropOutgoingHandshake:456(200ns)} GenerateSessionVirtualId:988(550ns) PrepareNewSessionHandshake:476(3915ns) {SwitchToState:178(1398ns)}}}}}
2025-11-11T17:19:37.085146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518723070830637:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.075075Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518717758140780:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.075119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518717758140780:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.086379Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518717758140788:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.086531Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518717758140780:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.090122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518721648706786:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
2025-11-11T17:19:37.073804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518719341979546:2078] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.073831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518719341979546:2078] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.091911Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518717353739475:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:37.093642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518718775863330:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.093379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518717353739474:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.093674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518718775863330:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.094738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518719341979546:2078] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:37.094808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:37.094850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518717353739474:2068] [node 4] ICP22 created new session: [5:7571518721648706788:2068]
2025-11-11T17:19:37.094894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518721648706788:2068] [node 4] ICS09 handshake done sender: [5:7571518717353739475:2111] self: [5:7571518721648706784:0] peer: [4:7571518717758140779:0] socket: 318 qp: -1
2025-11-11T17:19:37.094924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518721648706788:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:37.094951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.095719Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518718775863330:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:37.095815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518715748609316:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.092308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518717758140775:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.092362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:37.096211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518718775863317:2080] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.096262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.097767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518718775863317:2080] [node 4] ICP22 created new session: [2:7571518723070830641:2080]
2025-11-11T17:19:37.097823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518723070830641:2080] [node 4] ICS09 handshake done sender: [2:7571518718775863330:2111] self: [2:7571518718775863329:0] peer: [4:7571518722053108088:0] socket: 316 qp: -1
2025-11-11T17:19:37.097857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518723070830641:2080] [node 4] ICS10 traffic start
2025-11-11T17:19:37.097891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.092404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518717758140775:2067] [node 5] ICP22 created new session: [4:7571518722053108086:2067]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.102340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518721648706787:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
2025-11-11T17:19:37.103352Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518717353739476:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:37.098703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518716874623858:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30945044ns) {Configure:104(30940572ns) {ProcessConfigured:135(1387750ns) {StartInitialHandshake:157(1385069ns) {DropHandshakes:469(1279ns) {DropIncomingHandshake:441(284ns) DropOutgoingHandshake:456(234ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:476(2399ns) {SwitchToState:178(769ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.096681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576621:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.109214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518717353739410:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.092445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518722053108086:2067] [node 5] ICS09 handshake done sender: [4:7571518717758140780:2113] self: [4:7571518717758140779:0] peer: [5:7571518721648706784:0] socket: 309 qp: -1
2025-11-11T17:19:37.133179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518723636946888:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.133508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518719341979542:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.135451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:37.108589Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518720043576621:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.109265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:37.150494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518719341979542:2072] [node 5] ICP22 created new session: [3:7571518723636946892:2072]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.109305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518717353739410:2050] [node 3] ICP22 created new session: [5:7571518721648706791:2050]
2025-11-11T17:19:37.109354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518721648706791:2050] [node 3] ICS09 handshake done sender: [5:7571518717353739476:2112] self: [5:7571518721648706785:0] peer: [3:7571518719341979545:0] socket: 319 qp: -1
2025-11-11T17:19:37.113821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518721648706791:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:37.131909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:37.129637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518723070830643:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.092480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518722053108086:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:37.125782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518715748609262:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.150570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518723636946892:2072] [node 5] ICS09 handshake done sender: [3:7571518719341979546:2078] self: [3:7571518719341979545:0] peer: [5:7571518721648706785:0] socket: 310 qp: -1
2025-11-11T17:19:37.150629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518723636946892:2072] [node 5] ICS10 traffic start
2025-11-11T17:19:37.150681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:37.151159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518719341979542:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(17496052ns) {HandleHandshakeStatus:329(17493441ns) {DropIncomingHandshake:441(521ns) DropOutgoingHandshake:456(842ns) SwitchToState:178(671ns) ProcessPendingSessionEvents:495(111591ns) {ForwardSessionEventToSession:612(55537ns) ForwardSessionEventToSession:612(8949ns) ForwardSessionEventToSession:612(2718ns) ForwardSessionEventToSession:612(2481ns) ForwardSessionEventToSession:612(5103ns) ForwardSessionEventToSession:612(5161ns) ForwardSessionEventToSession:612(2812ns) ForwardSessionEventToSession:612(7699ns) ForwardSessionEventToSession:612(2419ns) ForwardSessionEventToSession:612(5163ns)}}}
2025-11-11T17:19:37.151836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518723636946891:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47448
2025-11-11T17:19:37.151987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518719341979512:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.133838Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518723070830643:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.169791Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518718775863332:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.169820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518718775863332:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.170748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518723070830646:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39515
2025-11-11T17:19:37.185723Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518718775863332:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:37.092516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.132537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518715748609262:2059] [node 5] ICP35 event processing took too much time PendingConnection:278(17127997ns) {IncomingHandshake:261(17125859ns) {DropIncomingHandshake:441(611ns) PrepareNewSessionHandshake:476(2537ns) {SwitchToState:178(887ns)} GenerateSessionVirtualId:988(727292ns) IssueIncomingHandshakeReply:193(48551ns) {SwitchToState:178(557ns)}}}
2025-11-11T17:19:37.093037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518717758140710:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.101675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518722053108090:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24060
2025-11-11T17:19:37.109781Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.165927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518717758140710:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518717758140788:2116] held: no
2025-11-11T17:19:37.166132Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518717758140788:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.203754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518717353739410:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(94025728ns) {HandleHandshakeStatus:329(94023947ns) {DropIncomingHandshake:441(457ns) DropOutgoingHandshake:456(4675ns) SwitchToState:178(128ns) ProcessPendingSessionEvents:495(57807260ns) {ForwardSessionEventToSession:612(25741064ns) ForwardSessionEventToSession:612(6840ns) ForwardSessionEventToSession:612(3024ns) ForwardSessionEventToSession:612(14719523ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(7824568ns) ForwardSessionEventToSession:612(7989288ns) ForwardSessionEventToSession:612(13464ns) ForwardSessionEventToSession:612(1473060ns) ForwardSessionEventToSession:612(2932ns)}}}
2025-11-11T17:19:37.203912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518721648706790:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
2025-11-11T17:19:37.204353Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518717353739480:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.204374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518717353739480:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.206126Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518723636946891:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.206240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518719341979544:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.206258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518719341979544:2077] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.207245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518723636946896:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47448
2025-11-11T17:19:37.202114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518715748609316:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.202465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576626:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.202625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518715748609251:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.210616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518716874623863:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36950
2025-11-11T17:19:37.210732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518716874623864:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36950
2025-11-11T17:19:37.210920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518721169591167:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36950
2025-11-11T17:19:37.211002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518721169591168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36950
2025-11-11T17:19:37.209159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518723070830646:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.207663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518717758140710:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.207717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:37.207758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518717758140710:2049] [node 2] ICP22 created new session: [4:7571518722053108091:2049]
2025-11-11T17:19:37.207804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518722053108091:2049] [node 2] ICS09 handshake done sender: [4:7571518717758140788:2116] self: [4:7571518722053108088:0] peer: [2:7571518718775863329:0] socket: 320 qp: -1
2025-11-11T17:19:37.207837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518722053108091:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:37.207870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.219728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518722053108094:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24060
2025-11-11T17:19:37.209206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518718775863262:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.209238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:37.209266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518718775863262:2049] [node 3] ICP22 created new session: [2:7571518723070830648:2049]
2025-11-11T17:19:37.220031Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518721169591168:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.209306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518723070830648:2049] [node 3] ICS09 handshake done sender: [2:7571518718775863332:2112] self: [2:7571518718775863331:0] peer: [3:7571518723636946895:0] socket: 317 qp: -1
2025-11-11T17:19:37.209336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518723070830648:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:37.213666Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518721648706790:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.234342Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.209358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:37.210072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518718775863262:2049] [node 3] ICP18 handshake (actor [2:7571518723070830646:2118]) is too old
2025-11-11T17:19:37.245861Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518716874623863:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.233682Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518717353739480:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:37.213654Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518720043576626:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.262269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518717353739473:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.262325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:37.262370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518717353739473:2066] [node 6] ICP22 created new session: [5:7571518721648706795:2066]
2025-11-11T17:19:37.262411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518721648706795:2066] [node 6] ICS09 handshake done sender: [5:7571518717353739480:2114] self: [5:7571518717353739479:0] peer: [6:7571518720043576625:0] socket: 325 qp: -1
2025-11-11T17:19:37.262442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518721648706795:2066] [node 6] ICS10 traffic start
2025-11-11T17:19:37.262478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:37.262726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518717353739473:2066] [node 6] ICP18 handshake (actor [5:7571518721648706790:2119]) is too old
2025-11-11T17:19:37.264492Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518722053108094:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.264597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518717758140709:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.264929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518721169591167:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.265033Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518716874623862:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.265053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518716874623862:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.270333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518721648706797:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
2025-11-11T17:19:37.270579Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518716874623862:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:37.270934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518716874623857:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.271032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:37.271076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518716874623857:2107] [node 4] ICP22 created new session: [1:7571518721169591176:2107]
2025-11-11T17:19:37.271123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518721169591176:2107] [node 4] ICS09 handshake done sender: [1:7571518716874623862:2112] self: [1:7571518716874623861:0] peer: [4:7571518722053108095:0] socket: 352 qp: -1
2025-11-11T17:19:37.271156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518721169591176:2107] [node 4] ICS10 traffic start
2025-11-11T17:19:37.271223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.272861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518721648706798:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.263728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576628:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.263895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518715748609250:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.263955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576629:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.274140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576630:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.274284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576631:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.274460Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25784) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.275254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518715748609262:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25784) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518720043576621:2116] held: no
2025-11-11T17:19:37.275411Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518720043576628:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.275466Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518720043576621:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:37.275706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518715748609262:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.275762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:37.275797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518715748609262:2059] [node 5] ICP22 created new session: [6:7571518720043576633:2059]
2025-11-11T17:19:37.275832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518720043576633:2059] [node 5] ICS09 handshake done sender: [6:7571518720043576621:2116] self: [6:7571518720043576625:0] peer: [5:7571518717353739479:0] socket: 332 qp: -1
2025-11-11T17:19:37.275859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518720043576633:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:37.275886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:37.276359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518715748609315:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.278602Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518716874623864:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.282087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518721648706798:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.282180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518717353739408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.282458Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518721169591164:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.282482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518721169591164:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.283376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518721648706800:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
2025-11-11T17:19:37.283580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518721169591164:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:37.284033Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518721648706798:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:37.284268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518716874623858:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.284336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:37.284372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518716874623858:2108] [node 5] ICP22 created new session: [1:7571518721169591181:2108]
2025-11-11T17:19:37.284414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518721169591181:2108] [node 5] ICS09 handshake done sender: [1:7571518721169591164:2116] self: [1:7571518721169591163:0] peer: [5:7571518721648706799:0] socket: 342 qp: -1
2025-11-11T17:19:37.284444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518721169591181:2108] [node 5] ICS10 traffic start
2025-11-11T17:19:37.284475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:37.260837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518719341979512:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518723636946888:2117] held: no
2025-11-11T17:19:37.289685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518720043576630:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.291086Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518718775863334:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.291131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518718775863334:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.293580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518718775863334:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:37.293908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518718775863323:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.293961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:37.294004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518718775863323:2084] [node 6] ICP22 created new session: [2:7571518723070830653:2084]
2025-11-11T17:19:37.294055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518723070830653:2084] [node 6] ICS09 handshake done sender: [2:7571518718775863334:2113] self: [2:7571518718775863333:0] peer: [6:7571518720043576632:0] socket: 340 qp: -1
2025-11-11T17:19:37.294084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518723070830653:2084] [node 6] ICS10 traffic start
2025-11-11T17:19:37.294113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:37.294967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518723070830652:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39515
2025-11-11T17:19:37.306046Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22713) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.307248Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.308254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518715748609251:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22713) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518715748609316:2111] held: no
2025-11-11T17:19:37.308548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518715748609250:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518720043576626:2117] held: no
2025-11-11T17:19:37.308648Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518715748609316:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:37.309050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518715748609249:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.309145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518715748609251:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.309219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:37.309270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518715748609251:2050] [node 3] ICP22 created new session: [6:7571518720043576639:2050]
2025-11-11T17:19:37.309310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518720043576639:2050] [node 3] ICS09 handshake done sender: [6:7571518715748609316:2111] self: [6:7571518720043576627:0] peer: [3:7571518719341979543:0] socket: 312 qp: -1
2025-11-11T17:19:37.309338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518720043576639:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:37.309373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:37.313739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518717353739408:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.313797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:37.313833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518717353739408:2048] [node 1] ICP22 created new session: [5:7571518721648706801:2048]
2025-11-11T17:19:37.313875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518721648706801:2048] [node 1] ICS09 handshake done sender: [5:7571518721648706798:2121] self: [5:7571518721648706799:0] peer: [1:7571518721169591163:0] socket: 360 qp: -1
2025-11-11T17:19:37.313905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518721648706801:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:37.313934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:37.261199Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518723636946888:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:37.290002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518721169591162:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.315291Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.318069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576637:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.319305Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518720043576626:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:37.319582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518715748609250:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.319668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:37.319710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518715748609250:2049] [node 2] ICP22 created new session: [6:7571518720043576642:2049]
2025-11-11T17:19:37.319771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518720043576642:2049] [node 2] ICS09 handshake done sender: [6:7571518720043576626:2117] self: [6:7571518720043576632:0] peer: [2:7571518718775863333:0] socket: 307 qp: -1
2025-11-11T17:19:37.319808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518720043576642:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:37.319836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:37.314239Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.279118Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518719341979544:2077] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:37.290029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518721169591162:2115] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.279382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518719341979512:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.338095Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518717758140784:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.279442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:37.279478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518719341979512:2049] [node 2] ICP22 created new session: [3:7571518723636946900:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.342116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576644:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.309711Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518721169591162:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:37.279518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518723636946900:2049] [node 2] ICS09 handshake done sender: [3:7571518723636946888:2117] self: [3:7571518723636946895:0] peer: [2:7571518718775863331:0] socket: 327 qp: -1
2025-11-11T17:19:37.279554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518723636946900:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:37.279581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:37.279967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518719341979541:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.280048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:37.280090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518719341979541:2071] [node 6] ICP22 created new session: [3:7571518723636946902:2071]
2025-11-11T17:19:37.280124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518723636946902:2071] [node 6] ICS09 handshake done sender: [3:7571518719341979544:2077] self: [3:7571518719341979543:0] peer: [6:7571518720043576627:0] socket: 306 qp: -1
2025-11-11T17:19:37.280151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518723636946902:2071] [node 6] ICS10 traffic start
2025-11-11T17:19:37.280175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:37.280598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518723636946898:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47448
2025-11-11T17:19:37.281582Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518723636946898:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.285856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518719341979511:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.293388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518723636946906:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47448
2025-11-11T17:19:37.310162Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518723636946898:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:37.310649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518719341979511:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.310744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:37.310810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518719341979511:2048] [node 1] ICP22 created new session: [3:7571518723636946907:2048]
2025-11-11T17:19:37.310849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518723636946907:2048] [node 1] ICS09 handshake done sender: [3:7571518723636946898:2120] self: [3:7571518723636946905:0] peer: [1:7571518721169591161:0] socket: 356 qp: -1
2025-11-11T17:19:37.310874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518723636946907:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:37.310909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:37.330274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518716874623793:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.330356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:37.330397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518716874623793:2049] [node 3] ICP22 created new session: [1:7571518721169591185:2049]
2025-11-11T17:19:37.330445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518721169591185:2049] [node 3] ICS09 handshake done sender: [1:7571518721169591162:2115] self: [1:7571518721169591161:0] peer: [3:7571518723636946905:0] socket: 353 qp: -1
2025-11-11T17:19:37.330482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518721169591185:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:37.330528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:37.338133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518717758140784:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.339187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518717758140709:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518722053108094:2118] held: no
2025-11-11T17:19:37.339578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518722053108096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24060
2025-11-11T17:19:37.366280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518722053108097:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24060
2025-11-11T17:19:37.366422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518722053108098:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24060
2025-11-11T17:19:37.366567Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518722053108094:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:37.367918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518716874623860:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.367951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518716874623860:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.369374Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518723070830652:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.367326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518717353739408:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:37.369496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518718775863261:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.373779Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518721648706797:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.369139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518721169591187:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36950
2025-11-11T17:19:37.377732Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518716874623860:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:37.373910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518717353739409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.383107Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518717758140784:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.389862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518720043576645:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61017
2025-11-11T17:19:37.390347Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518721169591166:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.398423Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.399577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518715748609249:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518720043576630:2120] held: no
2025-11-11T17:19:37.399725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518720043576630:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:37.400028Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518720043576628:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:37.390387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518721169591166:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.390962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518721169591187:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.381718Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518718775863336:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.400233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518715748609249:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.400299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:37.400344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518715748609249:2048] [node 1] ICP22 created new session: [6:7571518720043576646:2048]
2025-11-11T17:19:37.400388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518720043576646:2048] [node 1] ICS09 handshake done sender: [6:7571518720043576630:2120] self: [6:7571518720043576638:0] peer: [1:7571518716874623859:0] socket: 344 qp: -1
2025-11-11T17:19:37.400419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518720043576646:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:37.400451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:37.400835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518715748609315:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.400899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:37.400936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518715748609315:2065] [node 4] ICP22 created new session: [6:7571518720043576648:2065]
2025-11-11T17:19:37.400976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518720043576648:2065] [node 4] ICS09 handshake done sender: [6:7571518720043576628:2118] self: [6:7571518720043576636:0] peer: [4:7571518717758140783:0] socket: 349 qp: -1
2025-11-11T17:19:37.401008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518720043576648:2065] [node 4] ICS10 traffic start
2025-11-11T17:19:37.401053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:37.381758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518718775863336:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.395224Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518718775863336:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:37.391004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518716874623856:2106] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.423168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518721648706804:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47204
2025-11-11T17:19:37.422167Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:37.423676Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518721648706797:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:37.391100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:37.391145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518716874623856:2106] [node 6] ICP22 created new session: [1:7571518721169591189:2106]
2025-11-11T17:19:37.391217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518721169591189:2106] [node 6] ICS09 handshake done sender: [1:7571518716874623860:2111] self: [1:7571518716874623859:0] peer: [6:7571518720043576638:0] socket: 351 qp: -1
2025-11-11T17:19:37.391253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518721169591189:2106] [node 6] ICS10 traffic start
2025-11-11T17:19:37.391305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:37.392102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518716874623856:2106] [node 6] ICP18 handshake (actor [1:7571518721169591187:2120]) is too old
2025-11-11T17:19:37.392699Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518721169591166:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:37.392969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518716874623792:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.393052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:37.393092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518716874623792:2048] [node 2] ICP22 created new session: [1:7571518721169591192:2048]
2025-11-11T17:19:37.393133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518721169591192:2048] [node 2] ICS09 handshake done sender: [1:7571518721169591166:2117] self: [1:7571518721169591165:0] peer: [2:7571518723070830655:0] socket: 341 qp: -1
2025-11-11T17:19:37.393161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518721169591192:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:37.393216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:37.417975Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518722053108098:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.418048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518717758140709:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.418094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:37.418140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518717758140709:2048] [node 1] ICP22 created new session: [4:7571518722053108100:2048]
2025-11-11T17:19:37.418185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518722053108100:2048] [node 1] ICS09 handshake done sender: [4:7571518722053108094:2118] self: [4:7571518722053108095:0] peer: [1:7571518716874623861:0] socket: 354 qp: -1
2025-11-11T17:19:37.418228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518722053108100:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:37.418264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:37.418786Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518722053108097:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:37.418830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518717758140776:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.418859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:37.418926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518717758140776:2071] [node 6] ICP22 created new session: [4:7571518722053108102:2071]
2025-11-11T17:19:37.418969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518722053108102:2071] [node 6] ICS09 handshake done sender: [4:7571518717758140784:2115] self: [4:7571518717758140783:0] peer: [6:7571518720043576636:0] socket: 348 qp: -1
2025-11-11T17:19:37.418996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518722053108102:2071] [node 6] ICS10 traffic start
2025-11-11T17:19:37.419023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:37.419854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518717758140776:2071] [node 6] ICP18 handshake (actor [4:7571518722053108097:2120]) is too old
2025-11-11T17:19:37.419947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518717758140711:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:37.426021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518717353739409:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.426073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:37.426121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518717353739409:2049] [node 2] ICP22 created new session: [5:7571518721648706805:2049]
2025-11-11T17:19:37.426162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518721648706805:2049] [node 2] ICS09 handshake done sender: [5:7571518721648706797:2120] self: [5:7571518721648706803:0] peer: [2:7571518718775863335:0] socket: 359 qp: -1
2025-11-11T17:19:37.426198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518721648706805:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:37.426227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:37.430308Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518719341979586:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.430346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518719341979586:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:37.437840Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518719341979586:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:37.442277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518718775863316:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:37.442372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:37.442419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518718775863316:2078] [node 5] ICP22 created new session: [2:7571518723070830658:2078]
2025-11-11T17:19:37.442459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518723070830658:2078] [node 5] ICS09 handshake done sender: [2:7571518718775863336:2114] self: [2:7571518718775863335:0] peer: [5:7571518721648706803:0] socket: 339 qp: -1
2025-11-11T17:19:37.442494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518723070830658:2078] [node 5] ICS10 traffic start
2025-11-11T17:19:37.442549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:37.442097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518722053108105:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24060
2025-11-11T17:19:37.447418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518719341979584:2075] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:37.446664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518718775863261:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518723070830652:2119] held: no
2025-11-11T17:19:37.466282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518723070830657:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39515
2025-11-11T17:19:37.466752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518723070830652:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:37.466932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518718775863261:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.466971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:37.467024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518718775863261:2048] [node 1] ICP22 created new session: [2:7571518723070830660:2048]
2025-11-11T17:19:37.467066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518723070830660:2048] [node 1] ICS09 handshake done sender: [2:7571518723070830652:2119] self: [2:7571518723070830655:0] peer: [1:7571518721169591165:0] socket: 368 qp: -1
2025-11-11T17:19:37.467109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518723070830660:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:37.467146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:37.465764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:37.470249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518719341979584:2075] [node 4] ICP22 created new session: [3:7571518723636946910:2075]
2025-11-11T17:19:37.470297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518723636946910:2075] [node 4] ICS09 handshake done sender: [3:7571518719341979586:2114] self: [3:7571518719341979585:0] peer: [4:7571518722053108104:0] socket: 343 qp: -1
2025-11-11T17:19:37.470329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518723636946910:2075] [node 4] ICS10 traffic start
2025-11-11T17:19:37.470366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:37.469754Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518722053108098:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:37.470896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518717758140711:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:37.470950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:37.470991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518717758140711:2050] [node 3] ICP22 created new session: [4:7571518722053108106:2050]
2025-11-11T17:19:37.471029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518722053108106:2050] [node 3] ICS09 handshake done sender: [4:7571518722053108098:2121] self: [4:7571518722053108104:0] peer: [3:7571518719341979585:0] socket: 337 qp: -1
2025-11-11T17:19:37.471052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518722053108106:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:37.471075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:37.472585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518719341979584:2075] [node 4] ICP35 event processing took too much time PendingConnection:278(24990493ns) {HandleHandshakeStatus:329(24988866ns) {DropIncomingHandshake:441(13771ns) DropOutgoingHandshake:456(194ns) SwitchToState:178(591ns) ProcessPendingSessionEvents:495(1903157ns) {ForwardSessionEventToSession:612(1858824ns) ForwardSessionEventToSession:612(3986ns) ForwardSessionEventToSession:612(2760ns) ForwardSessionEventToSession:612(12023ns) ForwardSessionEventToSession:612(1908ns) ForwardSessionEventToSession:612(4815ns) ForwardSessionEventToSession:612(2482ns) ForwardSessionEventToSession:612(1818ns) ForwardSessionEventToSession:612(4235ns) ForwardSessionEventToSession:612(3036ns)}}}
2025-11-11T17:19:37.519550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:37.519807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518723636946907:2048] [node 1] ICS01 socket: 356 reason# EndOfStream
2025-11-11T17:19:37.519845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518719341979511:2048] [node 1] ICP30 unregister session Session# [3:7571518723636946907:2048] VirtualId# [3:7571518723636946905:0]
2025-11-11T17:19:37.519879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518723636946907:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.519969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:37.520019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:37.521677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:37.521931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518723070830660:2048] [node 1] ICS01 socket: 368 reason# EndOfStream
2025-11-11T17:19:37.521968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518718775863261:2048] [node 1] ICP30 unregister session Session# [2:7571518723070830660:2048] VirtualId# [2:7571518723070830655:0]
2025-11-11T17:19:37.521997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518723070830660:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.522067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:37.522103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:37.528175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:37.528430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518720043576646:2048] [node 1] ICS01 socket: 344 reason# EndOfStream
2025-11-11T17:19:37.528473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518715748609249:2048] [node 1] ICP30 unregister session Session# [6:7571518720043576646:2048] VirtualId# [6:7571518720043576638:0]
2025-11-11T17:19:37.528504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518720043576646:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.528582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:37.528639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:37.528993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:37.529186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518722053108100:2048] [node 1] ICS01 socket: 354 reason# EndOfStream
2025-11-11T17:19:37.529223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518717758140709:2048] [node 1] ICP30 unregister session Session# [4:7571518722053108100:2048] VirtualId# [4:7571518722053108095:0]
2025-11-11T17:19:37.529248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518722053108100:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.529307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:37.529341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:37.538000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:37.538241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518721648706801:2048] [node 1] ICS01 socket: 360 reason# EndOfStream
2025-11-11T17:19:37.538278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518717353739408:2048] [node 1] ICP30 unregister session Session# [5:7571518721648706801:2048] VirtualId# [5:7571518721648706799:0]
2025-11-11T17:19:37.538310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518721648706801:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.538413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:37.538452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:37.585777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:37.586018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518723636946900:2049] [node 2] ICS01 socket: 327 reason# EndOfStream
2025-11-11T17:19:37.586059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518719341979512:2049] [node 2] ICP30 unregister session Session# [3:7571518723636946900:2049] VirtualId# [3:7571518723636946895:0]
2025-11-11T17:19:37.586089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518723636946900:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.586156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:37.586196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:37.597736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:37.597735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:37.597976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518722053108091:2049] [node 2] ICS01 socket: 320 reason# EndOfStream
2025-11-11T17:19:37.598009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518717758140710:2049] [node 2] ICP30 unregister session Session# [4:7571518722053108091:2049] VirtualId# [4:7571518722053108088:0]
2025-11-11T17:19:37.597815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:37.598040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518722053108091:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.598004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518721648706805:2049] [node 2] ICS01 socket: 359 reason# EndOfStream
2025-11-11T17:19:37.598045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518717353739409:2049] [node 2] ICP30 unregister session Session# [5:7571518721648706805:2049] VirtualId# [5:7571518721648706803:0]
2025-11-11T17:19:37.598116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:37.598159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:37.598071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518721648706805:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.598151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:37.598189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:37.640655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518720043576642:2049] [node 2] ICS01 socket: 307 reason# EndOfStream
2025-11-11T17:19:37.640714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518715748609250:2049] [node 2] ICP30 unregister session Session# [6:7571518720043576642:2049] VirtualId# [6:7571518720043576632:0]
2025-11-11T17:19:37.640750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518720043576642:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.640854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:37.640890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:37.649748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:37.649968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518722053108106:2050] [node 3] ICS01 socket: 337 reason# EndOfStream
2025-11-11T17:19:37.650028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518717758140711:2050] [node 3] ICP30 unregister session Session# [4:7571518722053108106:2050] VirtualId# [4:7571518722053108104:0]
2025-11-11T17:19:37.650064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518722053108106:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.650146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:37.650184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:37.649749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:37.649825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:37.650039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518721648706791:2050] [node 3] ICS07 socket disconnect 319 reason# EndOfStream
2025-11-11T17:19:37.650066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518721648706791:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.650147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:37.650185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518721648706791:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:37.650887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518720043576639:2050] [node 3] ICS01 socket: 312 reason# EndOfStream
2025-11-11T17:19:37.650927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518715748609251:2050] [node 3] ICP30 unregister session Session# [6:7571518720043576639:2050] VirtualId# [6:7571518720043576627:0]
2025-11-11T17:19:37.650952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518720043576639:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.651006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:37.651041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:37.651468Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22713) Couldn't connect to any resolved address
2025-11-11T17:19:37.651751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518717353739410:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22713) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:37.651794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518721648706791:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:37.651827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518717353739410:2050] [node 3] ICP30 unregister session Session# [5:7571518721648706791:2050] VirtualId# [5:7571518721648706785:0]
2025-11-11T17:19:37.651864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:37.651893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518717353739410:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:37.651908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:37.661708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518717353739410:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:37.753776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:37.753990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518721648706788:2068] [node 4] ICS07 socket disconnect 318 reason# EndOfStream
2025-11-11T17:19:37.754017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518721648706788:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.754102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:37.754141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518721648706788:2068] [node 4] ICS15 start handshake
2025-11-11T17:19:37.754629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:37.754789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518720043576648:2065] [node 4] ICS01 socket: 349 reason# EndOfStream
2025-11-11T17:19:37.754825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518715748609315:2065] [node 4] ICP30 unregister session Session# [6:7571518720043576648:2065] VirtualId# [6:7571518720043576636:0]
2025-11-11T17:19:37.754854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518720043576648:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.754907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:37.754945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:37.755720Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64605) Couldn't connect to any resolved address
2025-11-11T17:19:37.755932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518717353739474:2068] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64605) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:37.755976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518721648706788:2068] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:37.756012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518717353739474:2068] [node 4] ICP30 unregister session Session# [5:7571518721648706788:2068] VirtualId# [5:7571518721648706784:0]
2025-11-11T17:19:37.756053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:37.756099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518717353739474:2068] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:37.756114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:37.765733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518717353739474:2068] [node 4] ICP33 wake up from error state
2025-11-11T17:19:37.815727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:37.815986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518720043576633:2059] [node 5] ICS01 socket: 332 reason# EndOfStream
2025-11-11T17:19:37.816029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518715748609262:2059] [node 5] ICP30 unregister session Session# [6:7571518720043576633:2059] VirtualId# [6:7571518720043576625:0]
2025-11-11T17:19:37.816063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518720043576633:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:37.816188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:37.816234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:37.992992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518722921455938:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:37.993045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518722921455939:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:38.037840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518726290382176:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:38.037894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518726290382177:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:38.105927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518725255970055:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:38.105973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518725255970056:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:38.172610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518728421527477:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:38.172650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518728421527478:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:38.172676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518728421527479:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:38.221862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518726910999809:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:38.221914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518726910999810:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:38.221943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518726910999811:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:38.462995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518726910999823:2060] [node 6] ICP01 ready to work
2025-11-11T17:19:38.433704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518727216423254:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:38.426500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518725255970068:2059] [node 4] ICP01 ready to work
2025-11-11T17:19:38.403489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518725486809720:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:38.403542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518725486809721:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:38.403571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518725486809722:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.509403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518728421527493:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61721
2025-11-11T17:19:38.493711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518727216423264:2065] [node 4] ICP01 ready to work
2025-11-11T17:19:38.492257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518726290382194:2062] [node 6] ICP01 ready to work
2025-11-11T17:19:38.518079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518726910999823:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(53157598ns) {Configure:104(53152056ns) {ProcessConfigured:135(61514ns) {StartInitialHandshake:157(58870ns) {DropHandshakes:469(3052ns) {DropIncomingHandshake:441(808ns) DropOutgoingHandshake:456(490ns)} GenerateSessionVirtualId:988(1004ns) PrepareNewSessionHandshake:476(1708ns) {SwitchToState:178(621ns)}}}}}
2025-11-11T17:19:38.518517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518726910999844:2066] [node 4] ICP01 ready to work
2025-11-11T17:19:38.541794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518725486809738:2062] [node 4] ICP01 ready to work
2025-11-11T17:19:38.597787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518726290382228:2063] [node 4] ICP01 ready to work
2025-11-11T17:19:38.590674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518728421527513:2074] [node 6] ICP01 ready to work
2025-11-11T17:19:38.682410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518725486809785:2075] [node 5] ICP01 ready to work
2025-11-11T17:19:38.701665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518726290382236:2071] [node 5] ICP01 ready to work
2025-11-11T17:19:38.591003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518728421527514:2080] [node 5] ICP01 ready to work
2025-11-11T17:19:38.789337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518725255970119:2070] [node 6] ICP01 ready to work
2025-11-11T17:19:38.699559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518728421527514:2080] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(106812335ns) {Configure:104(106808280ns) {ProcessConfigured:135(53689666ns) {StartInitialHandshake:157(53686005ns) {DropHandshakes:469(3589ns) {DropIncomingHandshake:441(745ns) DropOutgoingHandshake:456(794ns)} GenerateSessionVirtualId:988(777ns) PrepareNewSessionHandshake:476(5822ns) {SwitchToState:178(3001ns)}}}}}
2025-11-11T17:19:38.763324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518727216423254:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(185486332ns) {Configure:104(185475448ns) {ProcessConfigured:135(62886ns) {StartInitialHandshake:157(60723ns) {DropHandshakes:469(3163ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(668ns)} GenerateSessionVirtualId:988(802ns) PrepareNewSessionHandshake:476(4540ns) {SwitchToState:178(872ns)}}}}}
2025-11-11T17:19:38.862163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518728421527477:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(395378520ns) {Configure:104(395370255ns) {ProcessConfigured:135(72939ns) {StartInitialHandshake:157(70369ns) {DropHandshakes:469(3889ns) {DropIncomingHandshake:441(734ns) DropOutgoingHandshake:456(556ns)} GenerateSessionVirtualId:988(893ns) PrepareNewSessionHandshake:476(2643ns) {SwitchToState:178(1158ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.870657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518725486809738:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(272726387ns) {Configure:104(272719973ns) {ProcessConfigured:135(50065ns) {StartInitialHandshake:157(47589ns) {DropHandshakes:469(2104ns) {DropIncomingHandshake:441(921ns) DropOutgoingHandshake:456(291ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:476(1172ns) {SwitchToState:178(391ns)}}}}}
2025-11-11T17:19:38.882360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518725486809794:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:38.882570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518725486809795:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:38.761644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518727216423298:2062] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.933894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518728421527556:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61721
2025-11-11T17:19:38.905656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518725486809795:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.870071Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518722921455938:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(105008397ns) {Configure:104(105003842ns) {ProcessConfigured:135(68113ns) {StartInitialHandshake:157(64737ns) {DropHandshakes:469(3723ns) {DropIncomingHandshake:441(774ns) DropOutgoingHandshake:456(1015ns)} GenerateSessionVirtualId:988(963ns) PrepareNewSessionHandshake:476(2028ns) {SwitchToState:178(625ns)}}}}}
2025-11-11T17:19:38.921649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518725486809794:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.891403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518722921455939:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(81543622ns) {Configure:104(81536226ns) {ProcessConfigured:135(99339ns) {StartInitialHandshake:157(96713ns) {DropHandshakes:469(3883ns) {DropIncomingHandshake:441(752ns) DropOutgoingHandshake:456(1109ns)} GenerateSessionVirtualId:988(1134ns) PrepareNewSessionHandshake:476(2141ns) {SwitchToState:178(663ns)}}}}}
2025-11-11T17:19:38.893571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518727216423306:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-11T17:19:38.904761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518727216423307:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-11T17:19:38.933818Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518727216423307:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.941683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518725255970123:2075] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.950507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518726290382176:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(472351698ns) {Configure:104(472347784ns) {ProcessConfigured:135(50084ns) {StartInitialHandshake:157(47494ns) {DropHandshakes:469(2618ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(748ns)} GenerateSessionVirtualId:988(545ns) PrepareNewSessionHandshake:476(1133ns) {SwitchToState:178(445ns)}}}}}
2025-11-11T17:19:38.965842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518728421527556:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.965957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518728421527477:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.969117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518728421527558:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61721
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.974611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518726290382236:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21278699ns) {Configure:104(21275811ns) {ProcessConfigured:135(55518ns) {StartInitialHandshake:157(53301ns) {DropHandshakes:469(1237ns) {DropIncomingHandshake:441(266ns) DropOutgoingHandshake:456(192ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:476(696ns) {SwitchToState:178(146ns)}}}}}
2025-11-11T17:19:38.975326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518726290382244:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11013
2025-11-11T17:19:38.975498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518726290382245:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11013
2025-11-11T17:19:38.949913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518725486809785:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:38.993953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518725255970055:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(515966070ns) {Configure:104(508026891ns) {ProcessConfigured:135(11990529ns) {StartInitialHandshake:157(11987362ns) {DropHandshakes:469(2847ns) {DropIncomingHandshake:441(329ns) DropOutgoingHandshake:456(748ns)} GenerateSessionVirtualId:988(656ns) PrepareNewSessionHandshake:476(6155ns) {SwitchToState:178(3211ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.961784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518727216423306:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.949992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518725486809738:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.018419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518725255970126:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48173
2025-11-11T17:19:39.018573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518725255970127:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48173
2025-11-11T17:19:39.018674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518725255970128:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48173
2025-11-11T17:19:39.018768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518725255970129:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48173
2025-11-11T17:19:38.976520Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518726290382244:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.961869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518726910999844:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(396551131ns) {Configure:104(396547503ns) {ProcessConfigured:135(128332599ns) {StartInitialHandshake:157(128329267ns) {DropHandshakes:469(3517ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(607ns)} GenerateSessionVirtualId:988(473ns) PrepareNewSessionHandshake:476(5183ns) {SwitchToState:178(2758ns)}}}}}
2025-11-11T17:19:38.966313Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518727216423303:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.966334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518727216423303:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:38.967789Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518727216423301:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.967808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518727216423301:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:38.969407Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518727216423303:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:39.020543Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518727216423301:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:38.981632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518728421527518:2081] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.041669Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518725255970127:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.953888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518725486809800:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.022023Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518726290382245:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.962022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518726910999843:2078] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.981694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518728421527518:2081] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:38.998908Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518728421527556:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:38.999202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518728421527477:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:38.999269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:38.999326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518728421527477:2048] [node 1] ICP22 created new session: [4:7571518728421527560:2048]
2025-11-11T17:19:38.999368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518728421527560:2048] [node 1] ICS09 handshake done sender: [4:7571518728421527556:2116] self: [4:7571518728421527557:0] peer: [1:7571518727216423302:0] socket: 330 qp: -1
2025-11-11T17:19:38.999420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518728421527560:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:38.999450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:39.000080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518728421527559:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61721
2025-11-11T17:19:39.025729Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518728421527518:2081] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:39.050119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518728421527513:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.050167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:39.050210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518728421527513:2074] [node 6] ICP22 created new session: [4:7571518732716494860:2074]
2025-11-11T17:19:39.050255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518732716494860:2074] [node 6] ICS09 handshake done sender: [4:7571518728421527518:2081] self: [4:7571518728421527517:0] peer: [6:7571518725486809802:0] socket: 319 qp: -1
2025-11-11T17:19:39.050291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518732716494860:2074] [node 6] ICS10 traffic start
2025-11-11T17:19:39.050346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:39.061857Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518728421527559:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.033697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518727216423254:2063] [node 6] ICP35 event processing took too much time PendingConnection:278(31529027ns) {IncomingHandshake:261(20032362ns) {DropIncomingHandshake:441(713ns) PrepareNewSessionHandshake:476(3209ns) {SwitchToState:178(1049ns)} GenerateSessionVirtualId:988(911ns) IssueIncomingHandshakeReply:193(11366ns)}}
2025-11-11T17:19:39.056726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518727216423264:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.056791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:39.065675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518727216423264:2065] [node 4] ICP22 created new session: [1:7571518731511390615:2065]
2025-11-11T17:19:39.073661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518731511390615:2065] [node 4] ICS09 handshake done sender: [1:7571518727216423303:2111] self: [1:7571518727216423302:0] peer: [4:7571518728421527557:0] socket: 328 qp: -1
2025-11-11T17:19:38.962044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518726910999843:2078] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.070661Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518725255970128:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:38.965797Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518725486809800:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.070333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518726290382177:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(332461479ns) {Configure:104(308431703ns) {ProcessConfigured:135(61231ns) {StartInitialHandshake:157(58742ns) {DropHandshakes:469(4932ns) {DropIncomingHandshake:441(575ns) DropOutgoingHandshake:456(501ns)} GenerateSessionVirtualId:988(877ns) PrepareNewSessionHandshake:476(1797ns) {SwitchToState:178(516ns)}}}}}
2025-11-11T17:19:39.020802Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518726910999843:2078] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:39.081674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518731511390615:2065] [node 4] ICS10 traffic start
2025-11-11T17:19:39.065691Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518728421527493:2063] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.065869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518728421527478:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.073772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518728421527479:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:38.965933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518725486809720:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.081749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:39.042373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967184:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.094940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518725255970056:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(147229908ns) {Configure:104(147220744ns) {ProcessConfigured:135(15975456ns) {StartInitialHandshake:157(15971669ns) {DropHandshakes:469(9259795ns) {DropIncomingHandshake:441(1643ns) DropOutgoingHandshake:456(730ns)} GenerateSessionVirtualId:988(2499ns) PrepareNewSessionHandshake:476(4619ns) {SwitchToState:178(1928ns)}}}}}
2025-11-11T17:19:39.089633Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518726290382247:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.010219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518725486809805:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.085851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518731205967184:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.089682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518726290382247:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.098766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518730585349555:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11013
2025-11-11T17:19:39.099091Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518726290382247:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:39.099355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518726290382228:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.099413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:39.099459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518726290382228:2063] [node 4] ICP22 created new session: [2:7571518730585349557:2063]
2025-11-11T17:19:39.099499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518730585349557:2063] [node 4] ICS09 handshake done sender: [2:7571518726290382247:2113] self: [2:7571518726290382246:0] peer: [4:7571518732716494862:0] socket: 338 qp: -1
2025-11-11T17:19:39.099531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518730585349557:2063] [node 4] ICS10 traffic start
2025-11-11T17:19:39.099579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:39.138300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518730585349560:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11013
2025-11-11T17:19:39.109932Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518728421527531:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.096231Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518725255970126:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.096337Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518725255970070:2060] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.096357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518725255970070:2060] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.105788Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518725255970070:2060] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:39.109864Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518725255970129:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.149758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518725255970055:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.010294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518725486809806:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.106092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518727216423254:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.109976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518728421527531:2093] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.086040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518726910999844:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.088236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518726910999823:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.093676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:39.093748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518726910999823:2060] [node 6] ICP22 created new session: [5:7571518731205967186:2060]
2025-11-11T17:19:39.093804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518731205967186:2060] [node 6] ICS09 handshake done sender: [5:7571518726910999843:2078] self: [5:7571518726910999842:0] peer: [6:7571518725486809801:0] socket: 321 qp: -1
2025-11-11T17:19:39.093859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518731205967186:2060] [node 6] ICS10 traffic start
2025-11-11T17:19:39.093904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:39.138766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967189:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
2025-11-11T17:19:39.138864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967190:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
2025-11-11T17:19:39.153637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518731205967189:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.154260Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518731205967190:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.157801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518726910999809:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.165855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518726910999810:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.183764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967193:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
2025-11-11T17:19:39.153832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518730585349560:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.184811Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518731205967184:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.190435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518725255970068:2059] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.190520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:39.190559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518725255970068:2059] [node 4] ICP22 created new session: [3:7571518729550937439:2059]
2025-11-11T17:19:39.190603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518729550937439:2059] [node 4] ICS09 handshake done sender: [3:7571518725255970070:2060] self: [3:7571518725255970069:0] peer: [4:7571518732716494863:0] socket: 313 qp: -1
2025-11-11T17:19:39.190634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518729550937439:2059] [node 4] ICS10 traffic start
2025-11-11T17:19:39.190688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:39.191863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518729550937441:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48173
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.150290Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:39.134056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518725486809804:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.142251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518729781777103:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.158274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518729781777104:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.158405Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:39.162125Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518725486809805:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.162220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518725486809720:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518725486809800:2117] held: no
2025-11-11T17:19:39.162359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518725486809795:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:39.169958Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518725486809800:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:39.170238Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518725486809794:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:39.170505Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518729781777104:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.170562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518725486809785:2075] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.170636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:39.170686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518725486809785:2075] [node 5] ICP22 created new session: [6:7571518729781777105:2075]
2025-11-11T17:19:39.170731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518729781777105:2075] [node 5] ICS09 handshake done sender: [6:7571518725486809795:2114] self: [6:7571518725486809801:0] peer: [5:7571518726910999842:0] socket: 324 qp: -1
2025-11-11T17:19:39.170765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518729781777105:2075] [node 5] ICS10 traffic start
2025-11-11T17:19:39.170800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:39.171362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518725486809720:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.171423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:39.171462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518725486809720:2048] [node 1] ICP22 created new session: [6:7571518729781777107:2048]
2025-11-11T17:19:39.171499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518729781777107:2048] [node 1] ICS09 handshake done sender: [6:7571518725486809800:2117] self: [6:7571518725486809803:0] peer: [1:7571518727216423300:0] socket: 329 qp: -1
2025-11-11T17:19:39.171533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518729781777107:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:39.171563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:39.171826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518725486809738:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.171885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:39.171929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518725486809738:2062] [node 4] ICP22 created new session: [6:7571518729781777109:2062]
2025-11-11T17:19:39.171965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518729781777109:2062] [node 4] ICS09 handshake done sender: [6:7571518725486809794:2113] self: [6:7571518725486809802:0] peer: [4:7571518728421527517:0] socket: 320 qp: -1
2025-11-11T17:19:39.171994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518729781777109:2062] [node 4] ICS10 traffic start
2025-11-11T17:19:39.172017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:39.172497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518725486809721:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.189906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518725486809722:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.106182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.184969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518726910999844:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.185050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:39.185094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518726910999844:2066] [node 4] ICP22 created new session: [5:7571518731205967194:2066]
2025-11-11T17:19:39.185136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518731205967194:2066] [node 4] ICS09 handshake done sender: [5:7571518731205967184:2115] self: [5:7571518731205967185:0] peer: [4:7571518728421527524:0] socket: 354 qp: -1
2025-11-11T17:19:39.185168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518731205967194:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:39.185218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:39.206605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967196:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
2025-11-11T17:19:39.214324Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518725255970131:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.214360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518725255970131:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.215439Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518725255970129:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:39.153980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518726290382176:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.223376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518725255970055:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.223460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:39.223500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518725255970055:2048] [node 1] ICP22 created new session: [3:7571518729550937444:2048]
2025-11-11T17:19:39.223553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518729550937444:2048] [node 1] ICS09 handshake done sender: [3:7571518725255970129:2115] self: [3:7571518729550937438:0] peer: [1:7571518727216423308:0] socket: 345 qp: -1
2025-11-11T17:19:39.223588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518729550937444:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:39.223620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.182757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518728421527478:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518728421527559:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.226073Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518725255970131:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:39.106227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518727216423254:2063] [node 6] ICP22 created new session: [1:7571518731511390619:2063]
2025-11-11T17:19:39.226333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518729781777114:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.226653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518729781777115:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-11T17:19:39.226945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518725486809805:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:39.227100Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518729781777104:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:39.227239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518725486809721:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.227294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:39.227331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518725486809721:2049] [node 2] ICP22 created new session: [6:7571518729781777116:2049]
2025-11-11T17:19:39.227373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518729781777116:2049] [node 2] ICS09 handshake done sender: [6:7571518725486809805:2119] self: [6:7571518729781777111:0] peer: [2:7571518726290382248:0] socket: 341 qp: -1
2025-11-11T17:19:39.227402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518729781777116:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:39.227437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:39.227786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518725486809722:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.227832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:39.227870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518725486809722:2050] [node 3] ICP22 created new session: [6:7571518729781777118:2050]
2025-11-11T17:19:39.227907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518729781777118:2050] [node 3] ICS09 handshake done sender: [6:7571518729781777104:2122] self: [6:7571518729781777113:0] peer: [3:7571518725255970130:0] socket: 355 qp: -1
2025-11-11T17:19:39.227927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518729781777118:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:39.227953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:39.157646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518730585349555:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.173555Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518726290382249:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.173577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518726290382249:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.205625Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518726290382251:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.205675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518726290382251:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.223011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518730585349563:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11013
2025-11-11T17:19:39.225407Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518726290382249:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:39.225651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518726290382194:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.225733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:39.233744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518726290382194:2062] [node 6] ICP22 created new session: [2:7571518730585349565:2062]
2025-11-11T17:19:39.258472Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518730585349560:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:39.183383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518732716494865:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61721
2025-11-11T17:19:39.238159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518726910999844:2066] [node 4] ICP35 event processing took too much time PendingConnection:278(52845529ns) {HandleHandshakeStatus:329(52843921ns) {DropIncomingHandshake:441(1240ns) DropOutgoingHandshake:456(4573ns) SwitchToState:178(679ns) ProcessPendingSessionEvents:495(19870488ns) {ForwardSessionEventToSession:612(19796921ns) ForwardSessionEventToSession:612(7290ns) ForwardSessionEventToSession:612(8588ns) ForwardSessionEventToSession:612(3658ns) ForwardSessionEventToSession:612(19128ns) ForwardSessionEventToSession:612(3040ns) ForwardSessionEventToSession:612(3211ns) ForwardSessionEventToSession:612(11089ns) ForwardSessionEventToSession:612(2835ns) ForwardSessionEventToSession:612(2866ns)}}}
2025-11-11T17:19:39.184164Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518728421527559:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:39.184442Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518728421527531:2093] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:39.185462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518728421527478:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.185507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:39.185553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518728421527478:2049] [node 2] ICP22 created new session: [4:7571518732716494867:2049]
2025-11-11T17:19:39.185637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518732716494867:2049] [node 2] ICS09 handshake done sender: [4:7571518728421527559:2118] self: [4:7571518732716494862:0] peer: [2:7571518726290382246:0] socket: 347 qp: -1
2025-11-11T17:19:39.192352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518728421527514:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.192385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:39.192431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518728421527514:2080] [node 5] ICP22 created new session: [4:7571518732716494868:2080]
2025-11-11T17:19:39.192471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518732716494868:2080] [node 5] ICS09 handshake done sender: [4:7571518728421527531:2093] self: [4:7571518728421527524:0] peer: [5:7571518731205967185:0] socket: 348 qp: -1
2025-11-11T17:19:39.192494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518732716494868:2080] [node 5] ICS10 traffic start
2025-11-11T17:19:39.192526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:39.192979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518732716494864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61721
2025-11-11T17:19:39.193676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518732716494867:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:39.212022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:39.216014Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518728421527493:2063] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:39.216496Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:39.261765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518728421527479:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.261837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:39.261884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518728421527479:2050] [node 3] ICP22 created new session: [4:7571518732716494871:2050]
2025-11-11T17:19:39.261930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518732716494871:2050] [node 3] ICS09 handshake done sender: [4:7571518728421527493:2063] self: [4:7571518732716494863:0] peer: [3:7571518725255970069:0] socket: 314 qp: -1
2025-11-11T17:19:39.261970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518732716494871:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:39.262003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:39.262285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518728421527479:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:39.253754Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518731205967196:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.235234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518725255970119:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.106273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518731511390619:2063] [node 6] ICS09 handshake done sender: [1:7571518727216423301:2110] self: [1:7571518727216423300:0] peer: [6:7571518725486809803:0] socket: 327 qp: -1
2025-11-11T17:19:39.106302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518731511390619:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:39.106336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:39.107859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518731511390617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-11T17:19:39.107973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518731511390618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-11T17:19:39.126060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518727216423264:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(84149031ns) {HandleHandshakeStatus:329(84144370ns) {DropIncomingHandshake:441(1256ns) DropOutgoingHandshake:456(1414ns) SwitchToState:178(294ns) ProcessPendingSessionEvents:495(15866062ns) {ForwardSessionEventToSession:612(15802488ns) ForwardSessionEventToSession:612(7103ns) ForwardSessionEventToSession:612(13150ns) ForwardSessionEventToSession:612(5366ns) ForwardSessionEventToSession:612(5545ns) ForwardSessionEventToSession:612(2688ns) ForwardSessionEventToSession:612(2487ns) ForwardSessionEventToSession:612(5392ns) ForwardSessionEventToSession:612(1952ns) ForwardSessionEventToSession:612(3059ns)}}}
2025-11-11T17:19:39.127980Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518731511390618:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.128085Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518731511390617:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.166129Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518727216423309:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.166151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518727216423309:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.173881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518727216423309:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:39.193703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518727216423305:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.193735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518727216423305:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.210860Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518727216423311:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.210881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518727216423311:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.233652Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518727216423305:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:39.239828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518727216423311:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:39.240154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518722921455939:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.240239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:39.240302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518722921455939:2049] [node 3] ICP22 created new session: [1:7571518731511390628:2049]
2025-11-11T17:19:39.240357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518731511390628:2049] [node 3] ICS09 handshake done sender: [1:7571518727216423309:2115] self: [1:7571518727216423308:0] peer: [3:7571518729550937438:0] socket: 331 qp: -1
2025-11-11T17:19:39.240403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518731511390628:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:39.240441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:39.241089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518722921455938:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.241193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:39.241249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518722921455938:2048] [node 2] ICP22 created new session: [1:7571518731511390630:2048]
2025-11-11T17:19:39.241306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518731511390630:2048] [node 2] ICS09 handshake done sender: [1:7571518727216423305:2112] self: [1:7571518727216423304:0] peer: [2:7571518730585349561:0] socket: 364 qp: -1
2025-11-11T17:19:39.241345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518731511390630:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:39.241378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:39.269801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518730585349565:2062] [node 6] ICS09 handshake done sender: [2:7571518726290382249:2114] self: [2:7571518726290382248:0] peer: [6:7571518729781777111:0] socket: 339 qp: -1
2025-11-11T17:19:39.277776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518730585349565:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:39.289717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:39.290156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518726290382176:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.290222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:39.273678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967197:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
2025-11-11T17:19:39.290271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518726290382176:2048] [node 1] ICP22 created new session: [2:7571518730585349569:2048]
2025-11-11T17:19:39.290318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518730585349569:2048] [node 1] ICS09 handshake done sender: [2:7571518730585349560:2118] self: [2:7571518730585349561:0] peer: [1:7571518727216423304:0] socket: 366 qp: -1
2025-11-11T17:19:39.290353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518730585349569:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:39.290403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.235333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:39.235372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518725255970119:2070] [node 6] ICP22 created new session: [3:7571518729550937446:2070]
2025-11-11T17:19:39.235420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518729550937446:2070] [node 6] ICS09 handshake done sender: [3:7571518725255970131:2116] self: [3:7571518725255970130:0] peer: [6:7571518729781777113:0] socket: 352 qp: -1
2025-11-11T17:19:39.235454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518729550937446:2070] [node 6] ICS10 traffic start
2025-11-11T17:19:39.235510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:39.305751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518725255970133:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.305778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518725255970133:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.327930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518729550937450:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48173
2025-11-11T17:19:39.326111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518726290382194:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(100191285ns) {HandleHandshakeStatus:329(100188703ns) {DropIncomingHandshake:441(12373ns) DropOutgoingHandshake:456(215ns) SwitchToState:178(474ns) ProcessPendingSessionEvents:495(125182ns) {ForwardSessionEventToSession:612(73728ns) ForwardSessionEventToSession:612(7410ns) ForwardSessionEventToSession:612(2731ns) ForwardSessionEventToSession:612(5134ns) ForwardSessionEventToSession:612(2894ns) ForwardSessionEventToSession:612(6127ns) ForwardSessionEventToSession:612(4948ns) ForwardSessionEventToSession:612(3039ns) ForwardSessionEventToSession:612(3064ns) ForwardSessionEventToSession:612(6986ns)}}}
2025-11-11T17:19:39.293936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518727216423298:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.297724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518726910999811:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.286593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518728421527478:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(84260860ns) {HandleHandshakeStatus:329(84259256ns) {DropIncomingHandshake:441(321ns) DropOutgoingHandshake:456(132ns) SwitchToState:178(136ns) ProcessPendingSessionEvents:495(1526ns)}}
2025-11-11T17:19:39.329938Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518725255970133:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:39.294044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:39.327008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518730585349567:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11013
2025-11-11T17:19:39.298351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967199:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
2025-11-11T17:19:39.294087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518727216423298:2062] [node 5] ICP22 created new session: [1:7571518731511390632:2062]
2025-11-11T17:19:39.294131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518731511390632:2062] [node 5] ICS09 handshake done sender: [1:7571518727216423311:2116] self: [1:7571518727216423310:0] peer: [5:7571518731205967191:0] socket: 325 qp: -1
2025-11-11T17:19:39.294163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518731511390632:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:39.294198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:39.318301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518727216423298:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(24009645ns) {HandleHandshakeStatus:329(24006977ns) {DropIncomingHandshake:441(14875ns) DropOutgoingHandshake:456(600ns) SwitchToState:178(775ns) ProcessPendingSessionEvents:495(23634792ns) {ForwardSessionEventToSession:612(15571530ns) ForwardSessionEventToSession:612(5874ns) ForwardSessionEventToSession:612(8007719ns) ForwardSessionEventToSession:612(3111ns) ForwardSessionEventToSession:612(5688ns) ForwardSessionEventToSession:612(2488ns) ForwardSessionEventToSession:612(3634ns) ForwardSessionEventToSession:612(11419ns) ForwardSessionEventToSession:612(3678ns) ForwardSessionEventToSession:612(3873ns)}}}
2025-11-11T17:19:39.306789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518731205967190:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:39.318669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518726910999809:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.318763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:39.318813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518726910999809:2048] [node 1] ICP22 created new session: [5:7571518731205967201:2048]
2025-11-11T17:19:39.318860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518731205967201:2048] [node 1] ICS09 handshake done sender: [5:7571518731205967190:2117] self: [5:7571518731205967191:0] peer: [1:7571518727216423310:0] socket: 367 qp: -1
2025-11-11T17:19:39.318902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518731205967201:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:39.318940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:39.333866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518731205967200:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53591
2025-11-11T17:19:39.341979Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518731205967196:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:39.349877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518726910999811:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.349933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:39.349988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518726910999811:2050] [node 3] ICP22 created new session: [5:7571518731205967203:2050]
2025-11-11T17:19:39.350031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518731205967203:2050] [node 3] ICS09 handshake done sender: [5:7571518731205967196:2119] self: [5:7571518731205967198:0] peer: [3:7571518725255970132:0] socket: 371 qp: -1
2025-11-11T17:19:39.350068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518731205967203:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:39.350101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:39.360180Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518731205967189:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:39.373838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518726910999810:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.373945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:39.373995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518726910999810:2049] [node 2] ICP22 created new session: [5:7571518731205967205:2049]
2025-11-11T17:19:39.374043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518731205967205:2049] [node 2] ICS09 handshake done sender: [5:7571518731205967189:2116] self: [5:7571518731205967192:0] peer: [2:7571518726290382250:0] socket: 363 qp: -1
2025-11-11T17:19:39.374082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518731205967205:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:39.374141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:39.333775Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518730585349567:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.341300Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518726290382251:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:39.369970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518726290382236:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.370047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:39.370096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518726290382236:2071] [node 5] ICP22 created new session: [2:7571518730585349573:2071]
2025-11-11T17:19:39.370139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518730585349573:2071] [node 5] ICS09 handshake done sender: [2:7571518726290382251:2115] self: [2:7571518726290382250:0] peer: [5:7571518731205967192:0] socket: 333 qp: -1
2025-11-11T17:19:39.370176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518730585349573:2071] [node 5] ICS10 traffic start
2025-11-11T17:19:39.370213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:39.382029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518725255970123:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.382997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:39.383055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518725255970123:2075] [node 5] ICP22 created new session: [3:7571518729550937451:2075]
2025-11-11T17:19:39.383105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518729550937451:2075] [node 5] ICS09 handshake done sender: [3:7571518725255970133:2117] self: [3:7571518725255970132:0] peer: [5:7571518731205967198:0] socket: 370 qp: -1
2025-11-11T17:19:39.383141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518729550937451:2075] [node 5] ICS10 traffic start
2025-11-11T17:19:39.383179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:39.385642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518729550937450:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.385760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518725255970056:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:39.386075Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518730585349554:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:39.386104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518730585349554:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:39.386985Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518730585349554:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:39.387225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518726290382177:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:39.387293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:39.387334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518726290382177:2049] [node 3] ICP22 created new session: [2:7571518730585349576:2049]
2025-11-11T17:19:39.387378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518730585349576:2049] [node 3] ICS09 handshake done sender: [2:7571518730585349554:2116] self: [2:7571518730585349553:0] peer: [3:7571518729550937453:0] socket: 322 qp: -1
2025-11-11T17:19:39.387405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518730585349576:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:39.387443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:39.389662Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:39.390948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518725255970056:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518729550937450:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:39.399598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518729550937454:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48173
2025-11-11T17:19:39.402105Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518729550937450:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:39.402379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518725255970056:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:39.402451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:39.402498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518725255970056:2049] [node 2] ICP22 created new session: [3:7571518729550937455:2049]
2025-11-11T17:19:39.402538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518729550937455:2049] [node 2] ICS09 handshake done sender: [3:7571518729550937450:2120] self: [3:7571518729550937453:0] peer: [2:7571518730585349553:0] socket: 342 qp: -1
2025-11-11T17:19:39.402575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518729550937455:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:39.402610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:39.453820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:39.454081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518730585349569:2048] [node 1] ICS01 socket: 366 reason# EndOfStream
2025-11-11T17:19:39.454120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518726290382176:2048] [node 1] ICP30 unregister session Session# [2:7571518730585349569:2048] VirtualId# [2:7571518730585349561:0]
2025-11-11T17:19:39.454157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518730585349569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.454262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:39.454305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:39.464903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:39.465157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518729550937444:2048] [node 1] ICS01 socket: 345 reason# EndOfStream
2025-11-11T17:19:39.465199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518725255970055:2048] [node 1] ICP30 unregister session Session# [3:7571518729550937444:2048] VirtualId# [3:7571518729550937438:0]
2025-11-11T17:19:39.465231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518729550937444:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.465312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:39.465362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:39.469750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:39.470007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518729781777107:2048] [node 1] ICS01 socket: 329 reason# EndOfStream
2025-11-11T17:19:39.469872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:39.470038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518725486809720:2048] [node 1] ICP30 unregister session Session# [6:7571518729781777107:2048] VirtualId# [6:7571518725486809803:0]
2025-11-11T17:19:39.470105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518731205967201:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-11T17:19:39.470135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518726910999809:2048] [node 1] ICP30 unregister session Session# [5:7571518731205967201:2048] VirtualId# [5:7571518731205967191:0]
2025-11-11T17:19:39.470160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518731205967201:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.470069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518729781777107:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.469777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:39.470230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:39.470153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:39.470001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518728421527560:2048] [node 1] ICS01 socket: 330 reason# EndOfStream
2025-11-11T17:19:39.470041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518728421527477:2048] [node 1] ICP30 unregister session Session# [4:7571518728421527560:2048] VirtualId# [4:7571518728421527557:0]
2025-11-11T17:19:39.470070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518728421527560:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.470152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:39.470195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:39.470264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:39.470195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:39.612985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:39.613237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518729550937455:2049] [node 2] ICS01 socket: 342 reason# EndOfStream
2025-11-11T17:19:39.613273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518725255970056:2049] [node 2] ICP30 unregister session Session# [3:7571518729550937455:2049] VirtualId# [3:7571518729550937453:0]
2025-11-11T17:19:39.613305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518729550937455:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.613381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:39.613424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:39.619312Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:39.619529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518729781777116:2049] [node 2] ICS01 socket: 341 reason# EndOfStream
2025-11-11T17:19:39.619566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518725486809721:2049] [node 2] ICP30 unregister session Session# [6:7571518729781777116:2049] VirtualId# [6:7571518729781777111:0]
2025-11-11T17:19:39.619597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518729781777116:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.619683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:39.619725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:39.625755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:39.625988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518732716494867:2049] [node 2] ICS01 socket: 347 reason# EndOfStream
2025-11-11T17:19:39.626024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518728421527478:2049] [node 2] ICP30 unregister session Session# [4:7571518732716494867:2049] VirtualId# [4:7571518732716494862:0]
2025-11-11T17:19:39.626056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518732716494867:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.626148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:39.626186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:39.629708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:39.629925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518731205967205:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-11-11T17:19:39.629961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518726910999810:2049] [node 2] ICP30 unregister session Session# [5:7571518731205967205:2049] VirtualId# [5:7571518731205967192:0]
2025-11-11T17:19:39.629990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518731205967205:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.630087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:39.630137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:39.705800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:39.706048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518729781777118:2050] [node 3] ICS01 socket: 355 reason# EndOfStream
2025-11-11T17:19:39.706084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518725486809722:2050] [node 3] ICP30 unregister session Session# [6:7571518729781777118:2050] VirtualId# [6:7571518729781777113:0]
2025-11-11T17:19:39.706124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518729781777118:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.706221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:39.706262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:39.706335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:39.706516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518732716494871:2050] [node 3] ICS01 socket: 314 reason# EndOfStream
2025-11-11T17:19:39.706537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518728421527479:2050] [node 3] ICP30 unregister session Session# [4:7571518732716494871:2050] VirtualId# [4:7571518732716494863:0]
2025-11-11T17:19:39.706557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518732716494871:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.706600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:39.706632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:39.709734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:39.710064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518731205967203:2050] [node 3] ICS01 socket: 371 reason# EndOfStream
2025-11-11T17:19:39.710127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518726910999811:2050] [node 3] ICP30 unregister session Session# [5:7571518731205967203:2050] VirtualId# [5:7571518731205967198:0]
2025-11-11T17:19:39.710177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518731205967203:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.710318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:39.710385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:39.805826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:39.806084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518731205967194:2066] [node 4] ICS01 socket: 354 reason# EndOfStream
2025-11-11T17:19:39.806122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518726910999844:2066] [node 4] ICP30 unregister session Session# [5:7571518731205967194:2066] VirtualId# [5:7571518731205967185:0]
2025-11-11T17:19:39.806155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518731205967194:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.806238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:39.806275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:39.809728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:39.810014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518729781777109:2062] [node 4] ICS01 socket: 320 reason# EndOfStream
2025-11-11T17:19:39.810047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518725486809738:2062] [node 4] ICP30 unregister session Session# [6:7571518729781777109:2062] VirtualId# [6:7571518725486809802:0]
2025-11-11T17:19:39.810097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518729781777109:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.810235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:39.810308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:39.961771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:39.962022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518729781777105:2075] [node 5] ICS01 socket: 324 reason# EndOfStream
2025-11-11T17:19:39.962065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518725486809785:2075] [node 5] ICP30 unregister session Session# [6:7571518729781777105:2075] VirtualId# [6:7571518725486809801:0]
2025-11-11T17:19:39.962096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518729781777105:2075] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:39.962212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:39.962265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:40.197708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518736999605673:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:40.197792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518736999605674:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:40.262896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518734628211764:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:40.262949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518734628211765:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:40.305908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518733898094094:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:40.305953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518733898094095:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:40.373796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518733685777422:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:40.373838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518733685777423:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:40.373863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518733685777424:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:40.425829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518736269952423:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:40.425877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518736269952424:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:40.425908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518736269952425:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:40.517904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518735483998299:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:40.517950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518735483998300:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:40.517975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518735483998301:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:40.774947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518736269952436:2059] [node 6] ICP01 ready to work
2025-11-11T17:19:40.818257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518735483998317:2064] [node 5] ICP01 ready to work
2025-11-11T17:19:40.828964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518733898094106:2058] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:40.858111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518733898094116:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:40.859906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518733898094117:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24310
2025-11-11T17:19:40.883435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518733898094130:2074] [node 6] ICP01 ready to work
2025-11-11T17:19:40.825701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518736999605729:2095] [node 5] ICP01 ready to work
2025-11-11T17:19:40.833983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518734628211786:2068] [node 5] ICP01 ready to work
2025-11-11T17:19:40.849499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518736999605736:2096] [node 6] ICP01 ready to work
2025-11-11T17:19:40.958509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518733685777479:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:40.993640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518736999605738:2094] [node 4] ICP01 ready to work
2025-11-11T17:19:40.985790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518735483998350:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.030692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518736269952436:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(255402998ns) {Configure:104(255397118ns) {ProcessConfigured:135(28611788ns) {StartInitialHandshake:157(28609282ns) {DropHandshakes:469(2287ns) {DropIncomingHandshake:441(731ns) DropOutgoingHandshake:456(313ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:476(3987ns) {SwitchToState:178(1792ns)}}}}}
2025-11-11T17:19:41.046581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518738923179127:2065] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.027098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518735483998300:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(257250614ns) {Configure:104(257232057ns) {ProcessConfigured:135(55206ns) {StartInitialHandshake:157(52706ns) {DropHandshakes:469(3225ns) {DropIncomingHandshake:441(977ns) DropOutgoingHandshake:456(500ns)} GenerateSessionVirtualId:988(765ns) PrepareNewSessionHandshake:476(2123ns) {SwitchToState:178(571ns)}}}}}
2025-11-11T17:19:41.105398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518736999605673:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(198327174ns) {Configure:104(198320424ns) {ProcessConfigured:135(82089ns) {StartInitialHandshake:157(79373ns) {DropHandshakes:469(3488ns) {DropIncomingHandshake:441(744ns) DropOutgoingHandshake:456(689ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:476(2089ns) {SwitchToState:178(811ns)}}}}}
2025-11-11T17:19:41.032961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919790:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.062576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518734628211764:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(268562777ns) {Configure:104(268553716ns) {ProcessConfigured:135(54855ns) {StartInitialHandshake:157(52206ns) {DropHandshakes:469(3104ns) {DropIncomingHandshake:441(554ns) DropOutgoingHandshake:456(604ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:476(1559ns) {SwitchToState:178(598ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.074155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518738193061470:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24310
2025-11-11T17:19:41.063086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518738923179130:2079] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.033019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919791:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.097891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518737980744788:2061] [node 6] ICP01 ready to work
2025-11-11T17:19:41.150632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744791:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.150771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744792:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.082675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518733898094117:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.087229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518738923179131:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43557
2025-11-11T17:19:41.033211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919792:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.174419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518733685777422:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(332462636ns) {Configure:104(332457279ns) {ProcessConfigured:135(83077ns) {StartInitialHandshake:157(80115ns) {DropHandshakes:469(3677ns) {DropIncomingHandshake:441(1185ns) DropOutgoingHandshake:456(560ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:476(1719ns) {SwitchToState:178(560ns)}}}}}
2025-11-11T17:19:41.167181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518741294573043:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2420
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.171152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518736999605738:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(96986511ns) {Configure:104(96982729ns) {ProcessConfigured:135(8062862ns) {StartInitialHandshake:157(8059886ns) {DropHandshakes:469(7957380ns) {DropIncomingHandshake:441(860ns) DropOutgoingHandshake:456(3295ns)} GenerateSessionVirtualId:988(2808ns) PrepareNewSessionHandshake:476(2846ns) {SwitchToState:178(1340ns)}}}}}
2025-11-11T17:19:41.087402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518738923179132:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43557
2025-11-11T17:19:41.046496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518740564919795:2072] [node 4] ICP01 ready to work
2025-11-11T17:19:41.225925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518735483998299:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(183855883ns) {Configure:104(183849722ns) {ProcessConfigured:135(7659916ns) {StartInitialHandshake:157(7656928ns) {DropHandshakes:469(3390ns) {DropIncomingHandshake:441(755ns) DropOutgoingHandshake:456(643ns)} GenerateSessionVirtualId:988(947ns) PrepareNewSessionHandshake:476(2285ns) {SwitchToState:178(131ns)}}}}}
2025-11-11T17:19:41.226194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965674:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.174599Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518737980744792:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.137635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518733898094108:2059] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.085641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518740564919790:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.093658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518740564919791:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.094106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518736269952425:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.137788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518740564919792:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.161790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518736269952424:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.215490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919803:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.215948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919804:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.216059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919805:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.217064Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:41.218989Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518740564919791:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:41.219172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518736269952425:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518740564919791:2114] held: no
2025-11-11T17:19:41.219267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518736269952425:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.219326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:41.219371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518736269952425:2050] [node 3] ICP22 created new session: [5:7571518740564919806:2050]
2025-11-11T17:19:41.219411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518740564919806:2050] [node 3] ICS09 handshake done sender: [5:7571518740564919791:2114] self: [5:7571518740564919799:0] peer: [3:7571518733898094107:0] socket: 327 qp: -1
2025-11-11T17:19:41.219442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518740564919806:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:41.219472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:41.219828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518740564919792:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:41.220258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518736269952424:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.220305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:41.220345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518736269952424:2049] [node 2] ICP22 created new session: [5:7571518740564919808:2049]
2025-11-11T17:19:41.220382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518740564919808:2049] [node 2] ICS09 handshake done sender: [5:7571518740564919792:2115] self: [5:7571518740564919800:0] peer: [2:7571518734628211787:0] socket: 328 qp: -1
2025-11-11T17:19:41.220406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518740564919808:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:41.220434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:41.226438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518736269952423:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(376387242ns) {Configure:104(376381949ns) {ProcessConfigured:135(12474083ns) {StartInitialHandshake:157(12470232ns) {DropHandshakes:469(3036ns) {DropIncomingHandshake:441(501ns) DropOutgoingHandshake:456(633ns)} GenerateSessionVirtualId:988(1569ns) PrepareNewSessionHandshake:476(1655ns) {SwitchToState:178(178ns)}}}}}
2025-11-11T17:19:41.227247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919801:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.238255Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518740564919801:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.238481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518740564919795:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.087497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518738923179133:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.137678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518733898094108:2059] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.189198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518733685777423:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.158098Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518733898094108:2059] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:41.269718Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518741294573040:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.269769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518741294573040:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.270743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518741294573043:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.271158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518741294573047:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2420
2025-11-11T17:19:41.271244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518741294573048:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2420
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.189300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744796:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.109983Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179133:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.158254Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518738193061470:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.196224Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518737980744791:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.254407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965671:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.271990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518740564919804:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.278728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518736269952423:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.278815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919814:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.300124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518740564919787:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.300143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518740564919787:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.110059Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179131:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.280690Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518741294573040:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.166200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518738923179141:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43557
2025-11-11T17:19:41.265919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965675:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.193783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518738193061473:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24310
2025-11-11T17:19:41.310028Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518740564919787:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:41.298716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518736999605674:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(192959486ns) {Configure:104(192954171ns) {ProcessConfigured:135(19266767ns) {StartInitialHandshake:157(15154113ns) {DropHandshakes:469(2131ns) {DropIncomingHandshake:441(222ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(822ns) PrepareNewSessionHandshake:476(1943ns) {SwitchToState:178(238ns)}}}}}
2025-11-11T17:19:41.197977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518733685777424:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.167430Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518734628211788:2069] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.271439Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518739778965674:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.203138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518737980744788:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32827875ns) {Configure:104(32822108ns) {ProcessConfigured:135(78282ns) {StartInitialHandshake:157(74714ns) {DropHandshakes:469(3697ns) {DropIncomingHandshake:441(662ns) DropOutgoingHandshake:456(654ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:476(1821ns) {SwitchToState:178(590ns)}}}}}
2025-11-11T17:19:41.230432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.230616Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518737980744796:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.230729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518733685777422:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.249777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518733685777481:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.249805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518733685777481:2104] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.250459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744803:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.251207Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518737980744801:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.270500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744805:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.310943Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518733685777481:2104] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:41.314138Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:41.326402Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:41.331945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518733685777423:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518737980744792:2114] held: no
2025-11-11T17:19:41.332465Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518737980744791:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:41.332766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518733685777424:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518737980744791:2113] held: no
2025-11-11T17:19:41.332839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518733685777424:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.332873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:41.332924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518733685777424:2050] [node 3] ICP22 created new session: [4:7571518737980744808:2050]
2025-11-11T17:19:41.332959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518737980744808:2050] [node 3] ICS09 handshake done sender: [4:7571518737980744791:2113] self: [4:7571518737980744798:0] peer: [3:7571518733898094118:0] socket: 339 qp: -1
2025-11-11T17:19:41.332988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518737980744808:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:41.333015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:41.333379Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518737980744792:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:41.338972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518733685777423:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.366421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:41.369647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518733685777479:2059] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.298868Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518736999605742:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.310454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518740564919817:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37623
2025-11-11T17:19:41.213689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518733898094106:2058] [node 5] ICP35 event processing took too much time PendingConnection:278(19320637ns) {IncomingHandshake:261(19318288ns) {DropIncomingHandshake:441(665ns) PrepareNewSessionHandshake:476(7664452ns) {SwitchToState:178(3049ns)} GenerateSessionVirtualId:988(623ns) IssueIncomingHandshakeReply:193(13104ns)}}
2025-11-11T17:19:41.167453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518734628211788:2069] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.371679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518733685777423:2049] [node 2] ICP22 created new session: [4:7571518737980744810:2049]
2025-11-11T17:19:41.271530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518735483998299:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.379160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:41.330935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518736269952436:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.298891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518736999605742:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.214069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518733898094106:2058] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.168054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179132:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.331028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:41.405759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518733685777479:2059] [node 5] ICP22 created new session: [4:7571518737980744811:2059]
2025-11-11T17:19:41.214126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.299551Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518741294573047:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.283466Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518739778965675:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.185898Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518734628211788:2069] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:41.405824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518737980744810:2049] [node 2] ICS09 handshake done sender: [4:7571518737980744792:2114] self: [4:7571518737980744797:0] peer: [2:7571518738923179134:0] socket: 340 qp: -1
2025-11-11T17:19:41.214178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518733898094106:2058] [node 5] ICP22 created new session: [3:7571518738193061475:2058]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.301424Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518736999605740:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.297861Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518739778965671:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.205933Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179135:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.331077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518736269952436:2059] [node 6] ICP22 created new session: [5:7571518740564919818:2059]
2025-11-11T17:19:41.214233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518738193061475:2058] [node 5] ICS09 handshake done sender: [3:7571518733898094108:2059] self: [3:7571518733898094107:0] peer: [5:7571518740564919799:0] socket: 319 qp: -1
2025-11-11T17:19:41.214271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518738193061475:2058] [node 5] ICS10 traffic start
2025-11-11T17:19:41.214303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:41.225658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518738193061473:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.229755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518733898094119:2066] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.229792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518733898094119:2066] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.251178Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518733898094119:2066] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:41.278351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518738193061480:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24310
2025-11-11T17:19:41.281976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518733898094116:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.282040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:41.282082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518733898094116:2064] [node 4] ICP22 created new session: [3:7571518738193061481:2064]
2025-11-11T17:19:41.282133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518738193061481:2064] [node 4] ICS09 handshake done sender: [3:7571518733898094119:2066] self: [3:7571518733898094118:0] peer: [4:7571518737980744798:0] socket: 324 qp: -1
2025-11-11T17:19:41.282170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518738193061481:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:41.282207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:41.313776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518738193061480:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.325881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518733898094095:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.366123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518738193061485:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24310
2025-11-11T17:19:41.382245Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518738193061480:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:41.386110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518738193061486:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24310
2025-11-11T17:19:41.393840Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:41.418996Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518738193061486:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.419139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518733898094094:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.424299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518733898094095:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518738193061480:2118] held: no
2025-11-11T17:19:41.459246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518738193061488:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24310
2025-11-11T17:19:41.461322Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518738193061486:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:41.467083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518733898094094:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.467169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:41.467221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518733898094094:2048] [node 1] ICP22 created new session: [3:7571518738193061489:2048]
2025-11-11T17:19:41.467277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518738193061489:2048] [node 1] ICS09 handshake done sender: [3:7571518738193061486:2120] self: [3:7571518738193061487:0] peer: [1:7571518741294573051:0] socket: 377 qp: -1
2025-11-11T17:19:41.467314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518738193061489:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:41.467348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:41.475699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518733898094095:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(39407405ns) {HandleHandshakeStatus:395(32163063ns) {LogHandshakeFail:472(15563ns) {UpdateErrorStateLog:544(5160ns)} DropOutgoingHandshake:456(4883ns) ProcessPendingSessionEvents:495(12839342ns) {DropSessionEvent:512(35943ns) DropSessionEvent:512(11579ns) DropSessionEvent:512(4311ns) DropSessionEvent:512(5107ns) DropSessionEvent:512(4792ns) DropSessionEvent:512(3240ns) DropSessionEvent:512(3177ns) DropSessionEvent:512(6395829ns) DropSessionEvent:512(7606ns) DropSessionEvent:512(6358194ns)}}}
2025-11-11T17:19:41.475748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518733898094095:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.475785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:41.475838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518733898094095:2049] [node 2] ICP22 created new session: [3:7571518738193061491:2049]
2025-11-11T17:19:41.475876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518738193061491:2049] [node 2] ICS09 handshake done sender: [3:7571518738193061480:2118] self: [3:7571518738193061484:0] peer: [2:7571518738923179147:0] socket: 368 qp: -1
2025-11-11T17:19:41.475904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518738193061491:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:41.475941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:41.405868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518737980744810:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.301453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518736999605740:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.298092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518735483998317:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.205960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518738923179135:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.331121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518740564919818:2059] [node 6] ICS09 handshake done sender: [5:7571518740564919787:2111] self: [5:7571518740564919785:0] peer: [6:7571518739778965677:0] socket: 330 qp: -1
2025-11-11T17:19:41.405913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:41.206509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518734628211786:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.206593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:41.206641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518734628211786:2068] [node 5] ICP22 created new session: [2:7571518738923179144:2068]
2025-11-11T17:19:41.206682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518738923179144:2068] [node 5] ICS09 handshake done sender: [2:7571518734628211788:2069] self: [2:7571518734628211787:0] peer: [5:7571518740564919800:0] socket: 320 qp: -1
2025-11-11T17:19:41.206717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518738923179144:2068] [node 5] ICS10 traffic start
2025-11-11T17:19:41.206769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:41.234289Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179141:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.238903Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518738923179135:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:41.246890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518738923179130:2079] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.246933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:41.246980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518738923179130:2079] [node 4] ICP22 created new session: [2:7571518738923179149:2079]
2025-11-11T17:19:41.247021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518738923179149:2079] [node 4] ICS09 handshake done sender: [2:7571518738923179135:2115] self: [2:7571518738923179134:0] peer: [4:7571518737980744797:0] socket: 338 qp: -1
2025-11-11T17:19:41.247054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518738923179149:2079] [node 4] ICS10 traffic start
2025-11-11T17:19:41.247086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:41.253857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518738923179130:2079] [node 4] ICP18 handshake (actor [2:7571518738923179141:2117]) is too old
2025-11-11T17:19:41.269964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518734628211765:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(358093839ns) {Configure:104(358084178ns) {ProcessConfigured:135(36393543ns) {StartInitialHandshake:157(36389991ns) {DropHandshakes:469(16487712ns) {DropIncomingHandshake:441(1495ns) DropOutgoingHandshake:456(1062ns)} GenerateSessionVirtualId:988(2285ns) PrepareNewSessionHandshake:476(2459ns) {SwitchToState:178(684ns)}}}}}
2025-11-11T17:19:41.321945Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179137:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.321975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518738923179137:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.325355Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518738923179137:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:41.325623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518738923179127:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.325684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:41.325739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518738923179127:2065] [node 6] ICP22 created new session: [2:7571518738923179155:2065]
2025-11-11T17:19:41.325784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518738923179155:2065] [node 6] ICS09 handshake done sender: [2:7571518738923179137:2116] self: [2:7571518738923179136:0] peer: [6:7571518739778965678:0] socket: 362 qp: -1
2025-11-11T17:19:41.328618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518738923179155:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:41.334112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:41.337676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179151:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.337709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518738923179151:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.350643Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518738923179151:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:41.390804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518734628211765:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.390879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:41.390929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518734628211765:2049] [node 3] ICP22 created new session: [2:7571518738923179158:2049]
2025-11-11T17:19:41.390976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518738923179158:2049] [node 3] ICS09 handshake done sender: [2:7571518738923179151:2118] self: [2:7571518738923179147:0] peer: [3:7571518738193061484:0] socket: 366 qp: -1
2025-11-11T17:19:41.391017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518738923179158:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:41.391056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:41.398404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518738923179160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43557
2025-11-11T17:19:41.409751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518738923179160:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.417932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518734628211764:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.429720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518738923179127:2065] [node 6] ICP35 event processing took too much time PendingConnection:278(49848373ns) {HandleHandshakeStatus:329(49832749ns) {DropIncomingHandshake:441(13892ns) DropOutgoingHandshake:456(737ns) SwitchToState:178(425ns) ProcessPendingSessionEvents:495(13694595ns) {ForwardSessionEventToSession:612(13645304ns) ForwardSessionEventToSession:612(5048ns) ForwardSessionEventToSession:612(8357ns) ForwardSessionEventToSession:612(2788ns) ForwardSessionEventToSession:612(3093ns) ForwardSessionEventToSession:612(5347ns) ForwardSessionEventToSession:612(2306ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(4772ns) ForwardSessionEventToSession:612(2980ns)}}}
2025-11-11T17:19:41.468668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518738923179162:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43557
2025-11-11T17:19:41.470875Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518738923179160:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:41.471685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518734628211764:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.471731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:41.471770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518734628211764:2048] [node 1] ICP22 created new session: [2:7571518738923179163:2048]
2025-11-11T17:19:41.471813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518738923179163:2048] [node 1] ICS09 handshake done sender: [2:7571518738923179160:2119] self: [2:7571518738923179161:0] peer: [1:7571518741294573041:0] socket: 378 qp: -1
2025-11-11T17:19:41.471844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518738923179163:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:41.471870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:41.313711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518735483998300:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.509659Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518733898094132:2076] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.509730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518733898094132:2076] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.331153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518740564919818:2059] [node 6] ICS10 traffic start
2025-11-11T17:19:41.417677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518737980744811:2059] [node 5] ICS09 handshake done sender: [4:7571518733685777481:2104] self: [4:7571518733685777480:0] peer: [5:7571518740564919813:0] socket: 341 qp: -1
2025-11-11T17:19:41.354300Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518736999605740:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:41.318050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965679:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.354492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518736999605738:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.354529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:41.354580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518736999605738:2094] [node 4] ICP22 created new session: [1:7571518741294573056:2094]
2025-11-11T17:19:41.358306Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518736999605742:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:41.362105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518741294573056:2094] [node 4] ICS09 handshake done sender: [1:7571518741294573040:2112] self: [1:7571518741294573039:0] peer: [4:7571518737980744802:0] socket: 351 qp: -1
2025-11-11T17:19:41.362156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518741294573056:2094] [node 4] ICS10 traffic start
2025-11-11T17:19:41.362206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:41.362650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518736999605736:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.362691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:41.362727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518736999605736:2096] [node 6] ICP22 created new session: [1:7571518741294573058:2096]
2025-11-11T17:19:41.362772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518741294573058:2096] [node 6] ICS09 handshake done sender: [1:7571518736999605740:2110] self: [1:7571518736999605739:0] peer: [6:7571518739778965676:0] socket: 350 qp: -1
2025-11-11T17:19:41.362798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518741294573058:2096] [node 6] ICS10 traffic start
2025-11-11T17:19:41.362859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:41.373899Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518741294573048:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.421760Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518741294573052:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.421790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518741294573052:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.426094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518736999605729:2095] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.426153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:41.426199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518736999605729:2095] [node 5] ICP22 created new session: [1:7571518741294573063:2095]
2025-11-11T17:19:41.430131Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518741294573042:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.430155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518741294573042:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:41.441717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518741294573063:2095] [node 5] ICS09 handshake done sender: [1:7571518736999605742:2111] self: [1:7571518736999605741:0] peer: [5:7571518740564919815:0] socket: 349 qp: -1
2025-11-11T17:19:41.450136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518741294573063:2095] [node 5] ICS10 traffic start
2025-11-11T17:19:41.459469Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518741294573042:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:41.459753Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518741294573052:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:41.461527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518736999605673:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.461626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:41.461672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518736999605673:2048] [node 2] ICP22 created new session: [1:7571518741294573066:2048]
2025-11-11T17:19:41.461718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518741294573066:2048] [node 2] ICS09 handshake done sender: [1:7571518741294573042:2113] self: [1:7571518741294573041:0] peer: [2:7571518738923179161:0] socket: 375 qp: -1
2025-11-11T17:19:41.466643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518741294573066:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:41.469754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518736999605674:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.469838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:41.469879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518736999605674:2049] [node 3] ICP22 created new session: [1:7571518741294573070:2049]
2025-11-11T17:19:41.469918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518741294573070:2049] [node 3] ICS09 handshake done sender: [1:7571518741294573052:2117] self: [1:7571518741294573051:0] peer: [3:7571518738193061487:0] socket: 376 qp: -1
2025-11-11T17:19:41.469945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518741294573070:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:41.469976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:41.476940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518741294573068:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2420
2025-11-11T17:19:41.477004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518741294573069:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2420
2025-11-11T17:19:41.481951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:41.482032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:41.522028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518736999605729:2095] [node 5] ICP35 event processing took too much time PendingConnection:278(122933637ns) {HandleHandshakeStatus:329(122929522ns) {DropIncomingHandshake:441(13539ns) DropOutgoingHandshake:456(459ns) SwitchToState:178(594ns) ProcessPendingSessionEvents:495(11917816ns) {ForwardSessionEventToSession:612(11868754ns) ForwardSessionEventToSession:612(6181ns) ForwardSessionEventToSession:612(6416ns) ForwardSessionEventToSession:612(2195ns) ForwardSessionEventToSession:612(7647ns) ForwardSessionEventToSession:612(2451ns) ForwardSessionEventToSession:612(2462ns) ForwardSessionEventToSession:612(4588ns) ForwardSessionEventToSession:612(1752ns) ForwardSessionEventToSession:612(2709ns)}}}
2025-11-11T17:19:41.525446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518741294573069:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.531097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518736999605736:2096] [node 6] ICP18 handshake (actor [1:7571518741294573069:2119]) is too old
2025-11-11T17:19:41.553998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518736999605673:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(92240410ns) {HandleHandshakeStatus:329(92238213ns) {DropIncomingHandshake:441(13043ns) DropOutgoingHandshake:456(511ns) SwitchToState:178(721ns) ProcessPendingSessionEvents:495(22197559ns) {ForwardSessionEventToSession:612(22137966ns) ForwardSessionEventToSession:612(14911ns) ForwardSessionEventToSession:612(3303ns) ForwardSessionEventToSession:612(5079ns) ForwardSessionEventToSession:612(2760ns) ForwardSessionEventToSession:612(5017ns) ForwardSessionEventToSession:612(3543ns) ForwardSessionEventToSession:612(2154ns) ForwardSessionEventToSession:612(4770ns) ForwardSessionEventToSession:612(4737ns)}}}
2025-11-11T17:19:41.558113Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:41.331185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:41.561959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518736999605738:2094] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:41.425677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518737980744811:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:41.545726Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518733898094132:2076] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:41.332204Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518740564919801:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.378116Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:41.378972Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:41.392230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518735483998317:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518739778965671:2114] held: no
2025-11-11T17:19:41.392619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518735483998300:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518739778965675:2117] held: no
2025-11-11T17:19:41.392762Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518739778965671:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:41.393421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518735483998317:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.393457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:41.393497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518735483998317:2064] [node 5] ICP22 created new session: [6:7571518739778965683:2064]
2025-11-11T17:19:41.393542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518739778965683:2064] [node 5] ICS09 handshake done sender: [6:7571518739778965671:2114] self: [6:7571518739778965677:0] peer: [5:7571518740564919785:0] socket: 333 qp: -1
2025-11-11T17:19:41.393577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518739778965683:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:41.398852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:41.422049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965680:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.437946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965681:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.438133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965682:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.445976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518735483998317:2064] [node 5] ICP35 event processing took too much time PendingConnection:278(38898203ns) {HandleHandshakeStatus:329(38896451ns) {DropIncomingHandshake:441(709ns) DropOutgoingHandshake:456(135ns) SwitchToState:178(572ns) ProcessPendingSessionEvents:495(662ns)}}
2025-11-11T17:19:41.455390Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518739778965675:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:41.467944Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518739778965681:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.470943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518735483998301:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.474068Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518739778965674:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:41.485762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518735483998300:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.485852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:41.485903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518735483998300:2049] [node 2] ICP22 created new session: [6:7571518739778965688:2049]
2025-11-11T17:19:41.485941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518739778965688:2049] [node 2] ICS09 handshake done sender: [6:7571518739778965675:2117] self: [6:7571518739778965678:0] peer: [2:7571518738923179136:0] socket: 363 qp: -1
2025-11-11T17:19:41.485967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518739778965688:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:41.486022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:41.497188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965690:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.511237Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518739778965690:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.511286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518735483998299:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.511352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:41.511392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518735483998299:2048] [node 1] ICP22 created new session: [6:7571518739778965691:2048]
2025-11-11T17:19:41.511441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518739778965691:2048] [node 1] ICS09 handshake done sender: [6:7571518739778965674:2116] self: [6:7571518739778965676:0] peer: [1:7571518736999605739:0] socket: 353 qp: -1
2025-11-11T17:19:41.511483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518739778965691:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:41.511528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:41.549705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518735483998350:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:41.577756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518735483998350:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(18745426ns) {IncomingHandshake:261(10044303ns) {DropIncomingHandshake:441(606ns) PrepareNewSessionHandshake:476(2378ns) {SwitchToState:178(745ns)} GenerateSessionVirtualId:988(602ns) IssueIncomingHandshakeReply:193(64868ns) {SwitchToState:178(625ns)}}}
2025-11-11T17:19:41.577948Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:41.579149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518735483998301:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518739778965681:2120] held: no
2025-11-11T17:19:41.589996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965694:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.445876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:41.566052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518733898094130:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.566119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:41.597969Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518739778965681:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:41.347688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518740564919795:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.566170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518733898094130:2074] [node 6] ICP22 created new session: [3:7571518738193061494:2074]
2025-11-11T17:19:41.566213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518738193061494:2074] [node 6] ICS09 handshake done sender: [3:7571518733898094132:2076] self: [3:7571518733898094131:0] peer: [6:7571518739778965687:0] socket: 374 qp: -1
2025-11-11T17:19:41.566250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518738193061494:2074] [node 6] ICS10 traffic start
2025-11-11T17:19:41.566284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:41.373839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:41.375817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518740564919795:2072] [node 4] ICP22 created new session: [5:7571518740564919820:2072]
2025-11-11T17:19:41.417666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518740564919820:2072] [node 4] ICS09 handshake done sender: [5:7571518740564919801:2117] self: [5:7571518740564919813:0] peer: [4:7571518733685777480:0] socket: 346 qp: -1
2025-11-11T17:19:41.426265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518740564919820:2072] [node 4] ICS10 traffic start
2025-11-11T17:19:41.426312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:41.429910Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518740564919804:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:41.432631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518736269952423:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.461816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:41.466130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518740564919795:2072] [node 4] ICP35 event processing took too much time PendingConnection:278(131835038ns) {HandleHandshakeStatus:329(124234448ns) {DropIncomingHandshake:441(2019ns) DropOutgoingHandshake:456(11846ns) SwitchToState:178(645ns) ProcessPendingSessionEvents:495(133369ns) {ForwardSessionEventToSession:612(82303ns) ForwardSessionEventToSession:612(11210ns) ForwardSessionEventToSession:612(2381ns) ForwardSessionEventToSession:612(3068ns) ForwardSessionEventToSession:612(7983ns) ForwardSessionEventToSession:612(2857ns) ForwardSessionEventToSession:612(3345ns) ForwardSessionEventToSession:612(4685ns) ForwardSessionEventToSession:612(2502ns) ForwardSessionEventToSession:612(4407ns)}}}
2025-11-11T17:19:41.473945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518736269952423:2048] [node 1] ICP22 created new session: [5:7571518740564919822:2048]
2025-11-11T17:19:41.489738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518740564919822:2048] [node 1] ICS09 handshake done sender: [5:7571518740564919804:2119] self: [5:7571518740564919815:0] peer: [1:7571518736999605741:0] socket: 356 qp: -1
2025-11-11T17:19:41.489779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518740564919822:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:41.500366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:41.538850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518736269952423:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(105956197ns) {HandleHandshakeStatus:329(105953237ns) {DropIncomingHandshake:441(739ns) DropOutgoingHandshake:456(3577ns) SwitchToState:178(553ns) ProcessPendingSessionEvents:495(12154306ns) {ForwardSessionEventToSession:612(12097524ns) ForwardSessionEventToSession:612(9033ns) ForwardSessionEventToSession:612(5292ns) ForwardSessionEventToSession:612(2615ns) ForwardSessionEventToSession:612(3191ns) ForwardSessionEventToSession:612(5505ns) ForwardSessionEventToSession:612(3044ns) ForwardSessionEventToSession:612(2485ns) ForwardSessionEventToSession:612(5281ns) ForwardSessionEventToSession:612(8249ns)}}}
2025-11-11T17:19:41.457879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518733685777423:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(93438707ns) {HandleHandshakeStatus:329(93437366ns) {DropIncomingHandshake:441(1059ns) DropOutgoingHandshake:456(368ns) SwitchToState:178(563ns) ProcessPendingSessionEvents:495(938ns)}}
2025-11-11T17:19:41.460931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744816:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.466749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518737980744807:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-11T17:19:41.483828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518737980744796:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:41.484077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518733685777422:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.484126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:41.484171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518733685777422:2048] [node 1] ICP22 created new session: [4:7571518737980744817:2048]
2025-11-11T17:19:41.484222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518737980744817:2048] [node 1] ICS09 handshake done sender: [4:7571518737980744796:2116] self: [4:7571518737980744802:0] peer: [1:7571518741294573039:0] socket: 354 qp: -1
2025-11-11T17:19:41.484259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518737980744817:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:41.484289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:41.510367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518733685777479:2059] [node 5] ICP35 event processing took too much time PendingConnection:278(148459492ns) {HandleHandshakeStatus:329(148454962ns) {DropIncomingHandshake:441(22196ns) DropOutgoingHandshake:456(2122ns) SwitchToState:178(830ns) ProcessPendingSessionEvents:495(10026664ns) {ForwardSessionEventToSession:612(9976536ns) ForwardSessionEventToSession:612(3850ns) ForwardSessionEventToSession:612(6628ns) ForwardSessionEventToSession:612(3535ns) ForwardSessionEventToSession:612(2180ns) ForwardSessionEventToSession:612(7245ns) ForwardSessionEventToSession:612(6406ns) ForwardSessionEventToSession:612(4261ns) ForwardSessionEventToSession:612(2326ns) ForwardSessionEventToSession:612(3026ns)}}}
2025-11-11T17:19:41.526455Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518737980744816:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.573807Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518737980744800:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:41.573849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518737980744800:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:41.605560Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518737980744800:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:41.607924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518737980744788:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:41.607981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:41.608020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518737980744788:2061] [node 6] ICP22 created new session: [4:7571518737980744821:2061]
2025-11-11T17:19:41.608065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518737980744821:2061] [node 6] ICS09 handshake done sender: [4:7571518737980744800:2117] self: [4:7571518737980744799:0] peer: [6:7571518739778965693:0] socket: 336 qp: -1
2025-11-11T17:19:41.608095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518737980744821:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:41.608144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:41.602938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518735483998301:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.603024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:41.603076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518735483998301:2050] [node 3] ICP22 created new session: [6:7571518739778965695:2050]
2025-11-11T17:19:41.603140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518739778965695:2050] [node 3] ICS09 handshake done sender: [6:7571518739778965681:2120] self: [6:7571518739778965687:0] peer: [3:7571518733898094131:0] socket: 380 qp: -1
2025-11-11T17:19:41.603180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518739778965695:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:41.603220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:41.605372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518739778965697:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50006
2025-11-11T17:19:41.606069Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518739778965690:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:41.606262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518735483998350:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:41.606344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:41.606388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518735483998350:2061] [node 4] ICP22 created new session: [6:7571518739778965698:2061]
2025-11-11T17:19:41.606429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518739778965698:2061] [node 4] ICS09 handshake done sender: [6:7571518739778965690:2122] self: [6:7571518739778965693:0] peer: [4:7571518737980744799:0] socket: 332 qp: -1
2025-11-11T17:19:41.606462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518739778965698:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:41.606511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:41.716548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:41.716805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518738923179163:2048] [node 1] ICS01 socket: 378 reason# EndOfStream
2025-11-11T17:19:41.716842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518734628211764:2048] [node 1] ICP30 unregister session Session# [2:7571518738923179163:2048] VirtualId# [2:7571518738923179161:0]
2025-11-11T17:19:41.716877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518738923179163:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.716993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:41.717036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:41.720647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:41.725953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518738193061489:2048] [node 1] ICS07 socket disconnect 377 reason# EndOfStream
2025-11-11T17:19:41.725998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518738193061489:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.726115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:41.726152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518738193061489:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:41.733785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:41.734004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518739778965691:2048] [node 1] ICS01 socket: 353 reason# EndOfStream
2025-11-11T17:19:41.734045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518735483998299:2048] [node 1] ICP30 unregister session Session# [6:7571518739778965691:2048] VirtualId# [6:7571518739778965676:0]
2025-11-11T17:19:41.734077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518739778965691:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.734152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:41.734196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:41.737688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:41.735797Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address
2025-11-11T17:19:41.736126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518733898094094:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:41.736171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518738193061489:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:41.736200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518733898094094:2048] [node 1] ICP30 unregister session Session# [3:7571518738193061489:2048] VirtualId# [3:7571518738193061487:0]
2025-11-11T17:19:41.736247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:41.736284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518733898094094:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:41.736301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:41.737688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:41.737920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518737980744817:2048] [node 1] ICS01 socket: 354 reason# EndOfStream
2025-11-11T17:19:41.737956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518733685777422:2048] [node 1] ICP30 unregister session Session# [4:7571518737980744817:2048] VirtualId# [4:7571518737980744802:0]
2025-11-11T17:19:41.737984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518737980744817:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.738085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:41.738124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:41.737993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518740564919822:2048] [node 1] ICS07 socket disconnect 356 reason# EndOfStream
2025-11-11T17:19:41.738027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518740564919822:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.738101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:41.738140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518740564919822:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:41.738729Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address
2025-11-11T17:19:41.738926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518736269952423:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:41.738972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518740564919822:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:41.739001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518736269952423:2048] [node 1] ICP30 unregister session Session# [5:7571518740564919822:2048] VirtualId# [5:7571518740564919815:0]
2025-11-11T17:19:41.739036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:41.739060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518736269952423:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:41.739072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:41.745694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518733898094094:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:41.753718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518736269952423:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:41.836141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:41.836402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518738193061491:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-11-11T17:19:41.836467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518733898094095:2049] [node 2] ICP30 unregister session Session# [3:7571518738193061491:2049] VirtualId# [3:7571518738193061484:0]
2025-11-11T17:19:41.836511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518738193061491:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.836593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:41.836633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:41.854610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:41.854850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518740564919808:2049] [node 2] ICS01 socket: 328 reason# EndOfStream
2025-11-11T17:19:41.854889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518736269952424:2049] [node 2] ICP30 unregister session Session# [5:7571518740564919808:2049] VirtualId# [5:7571518740564919800:0]
2025-11-11T17:19:41.854926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518740564919808:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.855019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:41.855060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:41.857753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:41.858368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:41.858552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518739778965688:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-11-11T17:19:41.858583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518735483998300:2049] [node 2] ICP30 unregister session Session# [6:7571518739778965688:2049] VirtualId# [6:7571518739778965678:0]
2025-11-11T17:19:41.858610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518739778965688:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.858652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:41.858679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:41.857991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518737980744810:2049] [node 2] ICS01 socket: 340 reason# EndOfStream
2025-11-11T17:19:41.858029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518733685777423:2049] [node 2] ICP30 unregister session Session# [4:7571518737980744810:2049] VirtualId# [4:7571518737980744797:0]
2025-11-11T17:19:41.858061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518737980744810:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:41.858151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:41.858188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:42.004306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:42.004574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518737980744808:2050] [node 3] ICS01 socket: 339 reason# EndOfStream
2025-11-11T17:19:42.004607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518733685777424:2050] [node 3] ICP30 unregister session Session# [4:7571518737980744808:2050] VirtualId# [4:7571518737980744798:0]
2025-11-11T17:19:42.004636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518737980744808:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:42.004718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:42.004762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:42.004639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:42.004855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518739778965695:2050] [node 3] ICS01 socket: 380 reason# EndOfStream
2025-11-11T17:19:42.004891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518735483998301:2050] [node 3] ICP30 unregister session Session# [6:7571518739778965695:2050] VirtualId# [6:7571518739778965687:0]
2025-11-11T17:19:42.004915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518739778965695:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:42.004988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:42.005020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:42.005688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:42.005981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518740564919806:2050] [node 3] ICS01 socket: 327 reason# EndOfStream
2025-11-11T17:19:42.006037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518736269952425:2050] [node 3] ICP30 unregister session Session# [5:7571518740564919806:2050] VirtualId# [5:7571518740564919799:0]
2025-11-11T17:19:42.006076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518740564919806:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:42.006167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:42.006209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:42.049755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:42.049995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518740564919820:2072] [node 4] ICS01 socket: 346 reason# EndOfStream
2025-11-11T17:19:42.050029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518740564919795:2072] [node 4] ICP30 unregister session Session# [5:7571518740564919820:2072] VirtualId# [5:7571518740564919813:0]
2025-11-11T17:19:42.050058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518740564919820:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:42.050138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:42.050180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:42.050253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:42.050434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518739778965698:2061] [node 4] ICS01 socket: 332 reason# EndOfStream
2025-11-11T17:19:42.050468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518735483998350:2061] [node 4] ICP30 unregister session Session# [6:7571518739778965698:2061] VirtualId# [6:7571518739778965693:0]
2025-11-11T17:19:42.050487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518739778965698:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:42.050525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:42.050557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:42.133817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:42.134192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518739778965683:2064] [node 5] ICS01 socket: 333 reason# EndOfStream
2025-11-11T17:19:42.134267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518735483998317:2064] [node 5] ICP30 unregister session Session# [6:7571518739778965683:2064] VirtualId# [6:7571518739778965677:0]
2025-11-11T17:19:42.134328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518739778965683:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:42.134491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:42.134563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:42.289892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518743861417772:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:42.289942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518743861417773:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:42.361830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518745217204414:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:42.361879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518745217204415:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:42.454029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518745553200793:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:42.454070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518745553200794:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:42.544757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518743177311079:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:42.544805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518743177311080:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:42.544831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518743177311081:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:42.608222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518743559280025:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:42.608269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518743559280026:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:42.608315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518743559280027:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:42.664694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518742543492981:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:42.664734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518742543492982:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:42.664761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518742543492983:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:42.836424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518743559280040:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:42.814853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518745217204429:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:42.806520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518743861417796:2071] [node 4] ICP01 ready to work
2025-11-11T17:19:42.841726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518742543492998:2060] [node 5] ICP01 ready to work
2025-11-11T17:19:42.838625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518745217204435:2064] [node 6] ICP01 ready to work
2025-11-11T17:19:42.929506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518743861417773:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(120804595ns) {Configure:104(120800671ns) {ProcessConfigured:135(31566707ns) {StartInitialHandshake:157(31564115ns) {DropHandshakes:469(3590ns) {DropIncomingHandshake:441(902ns) DropOutgoingHandshake:456(692ns)} GenerateSessionVirtualId:988(1342ns) PrepareNewSessionHandshake:476(3469ns) {SwitchToState:178(2220ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:42.913745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518745553200810:2062] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:42.948400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518742543492999:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:42.937681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518745553200809:2059] [node 5] ICP01 ready to work
2025-11-11T17:19:42.840361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518745217204434:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5964
2025-11-11T17:19:43.008713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518747472278402:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:43.081156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518742543492999:2062] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.042934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518743559280040:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(206199213ns) {Configure:104(206193975ns) {ProcessConfigured:135(40947194ns) {StartInitialHandshake:157(40944127ns) {DropHandshakes:469(2940ns) {DropIncomingHandshake:441(642ns) DropOutgoingHandshake:456(614ns)} GenerateSessionVirtualId:988(783ns) PrepareNewSessionHandshake:476(5565ns) {SwitchToState:178(1814ns)}}}}}
2025-11-11T17:19:42.990546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518745217204458:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5964
2025-11-11T17:19:42.964150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518745553200822:2072] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.134410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518747854247384:2069] [node 6] ICP01 ready to work
2025-11-11T17:19:43.097677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518748156385128:2074] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.081309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518742543492982:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.026725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278403:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.117688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518748156385136:2073] [node 6] ICP01 ready to work
2025-11-11T17:19:43.018363Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518745217204434:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.143146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518742543492998:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(269238626ns) {Configure:104(269226446ns) {ProcessConfigured:135(83227ns) {StartInitialHandshake:157(80694ns) {DropHandshakes:469(3464ns) {DropIncomingHandshake:441(798ns) DropOutgoingHandshake:456(757ns)} GenerateSessionVirtualId:988(1395ns) PrepareNewSessionHandshake:476(1985ns) {SwitchToState:178(717ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.282208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518745553200794:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(440202637ns) {Configure:104(440198391ns) {ProcessConfigured:135(57825ns) {StartInitialHandshake:157(54758ns) {DropHandshakes:469(3062ns) {DropIncomingHandshake:441(827ns) DropOutgoingHandshake:456(631ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:476(2349ns) {SwitchToState:178(950ns)}}}}}
2025-11-11T17:19:43.284311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518749848168159:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46887
2025-11-11T17:19:43.034645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518745217204458:2086] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.033992Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518747472278403:2070] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.302995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518745553200810:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(317071635ns) {Configure:104(317064927ns) {ProcessConfigured:135(63096ns) {StartInitialHandshake:157(59787ns) {DropHandshakes:469(3356ns) {DropIncomingHandshake:441(745ns) DropOutgoingHandshake:456(496ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:476(3152ns) {SwitchToState:178(1039ns)}}}}}
2025-11-11T17:19:43.187156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518743861417812:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4657
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.034743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518745217204431:2062] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.219771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518746838460345:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.187363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518748156385139:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4657
2025-11-11T17:19:43.034379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518743177311080:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.303867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460354:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.335016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518745553200822:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(288223084ns) {Configure:104(288215667ns) {ProcessConfigured:135(65199ns) {StartInitialHandshake:157(63438ns) {DropHandshakes:469(1929ns) {DropIncomingHandshake:441(301ns) DropOutgoingHandshake:456(360ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:476(1492ns) {SwitchToState:178(710ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.034762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518745217204431:2062] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.085707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278405:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.225852Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518743861417812:2085] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.322115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518743559280025:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(432164029ns) {Configure:104(432159527ns) {ProcessConfigured:135(57985ns) {StartInitialHandshake:157(55769ns) {DropHandshakes:469(1152ns) {DropIncomingHandshake:441(266ns) DropOutgoingHandshake:456(191ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:476(2021ns) {SwitchToState:178(602ns)}}}}}
2025-11-11T17:19:43.304332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460355:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.323920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247393:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.347454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247397:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.347571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247398:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.347676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247392:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.366634Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518747854247393:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.383469Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518747854247392:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.414336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518745553200793:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(448351623ns) {Configure:104(448344767ns) {ProcessConfigured:135(19154547ns) {StartInitialHandshake:157(12231697ns) {DropHandshakes:469(3913ns) {DropIncomingHandshake:441(445ns) DropOutgoingHandshake:456(682ns)} GenerateSessionVirtualId:988(968ns) PrepareNewSessionHandshake:476(3743ns) {SwitchToState:178(1891ns)}}}}}
2025-11-11T17:19:43.219897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518747472278410:2062] [node 6] ICP01 ready to work
2025-11-11T17:19:43.035778Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518745217204431:2062] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:43.265659Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518748156385139:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.383637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518743559280025:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.304485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460353:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.036003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518745217204429:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.224451Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518747472278403:2070] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:43.358098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518743861417772:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(471182030ns) {Configure:104(471175703ns) {ProcessConfigured:135(24162623ns) {StartInitialHandshake:157(24159624ns) {DropHandshakes:469(7394935ns) {DropIncomingHandshake:441(316ns) DropOutgoingHandshake:456(2344ns)} GenerateSessionVirtualId:988(2739ns) PrepareNewSessionHandshake:476(4505ns) {SwitchToState:178(1670ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.390348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247402:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.358595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518748156385151:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4657
2025-11-11T17:19:43.036061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.307368Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518742543492999:2062] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:43.247018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518743177311080:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.460703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518749848168163:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.407027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518747854247384:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(188145063ns) {Configure:104(188138175ns) {ProcessConfigured:135(64969ns) {StartInitialHandshake:157(63067ns) {DropHandshakes:469(2359ns) {DropIncomingHandshake:441(513ns) DropOutgoingHandshake:456(316ns)} GenerateSessionVirtualId:988(693ns) PrepareNewSessionHandshake:476(2635ns) {SwitchToState:178(849ns)}}}}}
2025-11-11T17:19:43.036101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518745217204429:2061] [node 4] ICP22 created new session: [2:7571518749512171767:2061]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.247073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:43.247122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518743177311080:2049] [node 2] ICP22 created new session: [4:7571518747472278446:2049]
2025-11-11T17:19:43.247166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518747472278446:2049] [node 2] ICS09 handshake done sender: [4:7571518747472278403:2070] self: [4:7571518747472278404:0] peer: [2:7571518745217204430:0] socket: 334 qp: -1
2025-11-11T17:19:43.247206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518747472278446:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:43.247238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:43.247849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278438:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.307557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518742543492982:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.249137Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.460740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518749848168163:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.409735Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518747854247398:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.375077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518743861417798:2072] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.307637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:43.307679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518742543492982:2049] [node 2] ICP22 created new session: [6:7571518746838460356:2049]
2025-11-11T17:19:43.307723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518746838460356:2049] [node 2] ICS09 handshake done sender: [6:7571518742543492999:2062] self: [6:7571518746838460339:0] peer: [2:7571518745217204436:0] socket: 331 qp: -1
2025-11-11T17:19:43.307756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518746838460356:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:43.307795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:43.315616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460358:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.333755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518746838460355:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.402607Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518746838460353:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.415394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518742543492983:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(409916310ns) {Configure:104(409911226ns) {ProcessConfigured:135(12677851ns) {StartInitialHandshake:157(12675442ns) {DropHandshakes:469(1343ns) {DropIncomingHandshake:441(323ns) DropOutgoingHandshake:456(179ns)} GenerateSessionVirtualId:988(411ns) PrepareNewSessionHandshake:476(2917ns) {SwitchToState:178(1257ns)}}}}}
2025-11-11T17:19:43.426574Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518746838460345:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(188888930ns) {Configure:104(188872390ns) {ProcessConfigured:135(57303ns) {StartInitialHandshake:157(54905ns) {DropHandshakes:469(3012ns) {DropIncomingHandshake:441(325ns) DropOutgoingHandshake:456(1172ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:476(1807ns) {SwitchToState:178(864ns)}}}}}
2025-11-11T17:19:43.426779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518746838460345:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.433792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518742543492981:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.434013Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518746838460358:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.453772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518742543492983:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.485839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518742543492983:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(22257827ns) {IncomingHandshake:261(22255782ns) {DropIncomingHandshake:441(548ns) PrepareNewSessionHandshake:476(2026ns) {SwitchToState:178(712ns)} GenerateSessionVirtualId:988(503ns) IssueIncomingHandshakeReply:193(15159655ns) {SwitchToState:178(5813ns)}}}
2025-11-11T17:19:43.499655Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.500638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518742543492981:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518746838460353:2114] held: no
2025-11-11T17:19:43.526781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460368:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.036144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518749512171767:2061] [node 4] ICS09 handshake done sender: [2:7571518745217204431:2062] self: [2:7571518745217204430:0] peer: [4:7571518747472278404:0] socket: 328 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.375126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518743861417798:2072] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.427360Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518743559280026:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(513444797ns) {Configure:104(513441288ns) {ProcessConfigured:135(17358411ns) {StartInitialHandshake:157(17356542ns) {DropHandshakes:469(3133ns) {DropIncomingHandshake:441(737ns) DropOutgoingHandshake:456(673ns)} GenerateSessionVirtualId:988(656ns) PrepareNewSessionHandshake:476(2322ns) {SwitchToState:178(661ns)}}}}}
2025-11-11T17:19:43.386731Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518748156385141:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.386755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518748156385141:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.387935Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518743861417798:2072] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:43.389128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518748156385141:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:43.389574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518748156385128:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.389664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:43.391592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518743861417796:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.391656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:43.394892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518743861417796:2071] [node 4] ICP22 created new session: [1:7571518748156385155:2071]
2025-11-11T17:19:43.395125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518748156385128:2074] [node 5] ICP22 created new session: [1:7571518748156385154:2074]
2025-11-11T17:19:43.398837Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518748156385151:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.415826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518748156385154:2074] [node 5] ICS09 handshake done sender: [1:7571518748156385141:2114] self: [1:7571518748156385140:0] peer: [5:7571518747854247401:0] socket: 340 qp: -1
2025-11-11T17:19:43.422434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518748156385154:2074] [node 5] ICS10 traffic start
2025-11-11T17:19:43.427425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518748156385155:2071] [node 4] ICS09 handshake done sender: [1:7571518743861417798:2072] self: [1:7571518743861417797:0] peer: [4:7571518747472278459:0] socket: 347 qp: -1
2025-11-11T17:19:43.427461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518748156385155:2071] [node 4] ICS10 traffic start
2025-11-11T17:19:43.438563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:43.438683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:43.444783Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518748156385143:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.444803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518748156385143:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.470232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518748156385160:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4657
2025-11-11T17:19:43.482498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518743861417796:2071] [node 4] ICP35 event processing took too much time PendingConnection:278(90639501ns) {HandleHandshakeStatus:329(90636864ns) {DropIncomingHandshake:441(13186ns) DropOutgoingHandshake:456(313ns) SwitchToState:178(646ns) ProcessPendingSessionEvents:495(121751ns) {ForwardSessionEventToSession:612(64360ns) ForwardSessionEventToSession:612(6906ns) ForwardSessionEventToSession:612(5804ns) ForwardSessionEventToSession:612(5571ns) ForwardSessionEventToSession:612(3102ns) ForwardSessionEventToSession:612(2137ns) ForwardSessionEventToSession:612(5216ns) ForwardSessionEventToSession:612(3460ns) ForwardSessionEventToSession:612(2413ns) ForwardSessionEventToSession:612(5323ns)}}}
2025-11-11T17:19:43.485072Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518748156385143:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:43.494598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518748156385160:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.494651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518748156385136:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.494710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:43.494752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518748156385136:2073] [node 6] ICP22 created new session: [1:7571518748156385162:2073]
2025-11-11T17:19:43.494811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518748156385162:2073] [node 6] ICS09 handshake done sender: [1:7571518748156385143:2115] self: [1:7571518748156385142:0] peer: [6:7571518746838460365:0] socket: 341 qp: -1
2025-11-11T17:19:43.494849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518748156385162:2073] [node 6] ICS10 traffic start
2025-11-11T17:19:43.494890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:43.502910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518748156385128:2074] [node 5] ICP35 event processing took too much time PendingConnection:278(113121956ns) {HandleHandshakeStatus:329(113118060ns) {DropIncomingHandshake:441(698ns) DropOutgoingHandshake:456(987ns) SwitchToState:178(803ns) ProcessPendingSessionEvents:495(17529950ns) {ForwardSessionEventToSession:612(17476986ns) ForwardSessionEventToSession:612(10451ns) ForwardSessionEventToSession:612(3087ns) ForwardSessionEventToSession:612(5149ns) ForwardSessionEventToSession:612(2416ns) ForwardSessionEventToSession:612(2154ns) ForwardSessionEventToSession:612(4340ns) ForwardSessionEventToSession:612(2455ns) ForwardSessionEventToSession:612(5437ns) ForwardSessionEventToSession:612(5051ns)}}}
2025-11-11T17:19:43.502989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518748156385128:2074] [node 5] ICP18 handshake (actor [1:7571518748156385151:2117]) is too old
2025-11-11T17:19:43.541915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518748156385150:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.541945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518748156385150:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.552205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518743861417811:2084] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.552229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518743861417811:2084] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.553097Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518743861417811:2084] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:43.553345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518743861417773:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.553424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:43.553471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518743861417773:2049] [node 3] ICP22 created new session: [1:7571518748156385167:2049]
2025-11-11T17:19:43.553519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518748156385167:2049] [node 3] ICS09 handshake done sender: [1:7571518743861417811:2084] self: [1:7571518743861417809:0] peer: [3:7571518749848168173:0] socket: 330 qp: -1
2025-11-11T17:19:43.553551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518748156385167:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:43.553583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:43.259040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518743177311080:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:43.259372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278451:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.302072Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518747472278451:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.314760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518747472278402:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(305629777ns) {Configure:104(305623949ns) {ProcessConfigured:135(36194219ns) {StartInitialHandshake:157(36190974ns) {DropHandshakes:469(3276ns) {DropIncomingHandshake:441(884ns) DropOutgoingHandshake:456(685ns)} GenerateSessionVirtualId:988(1096ns) PrepareNewSessionHandshake:476(3120ns) {SwitchToState:178(1391ns)}}}}}
2025-11-11T17:19:43.337136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518743177311079:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.366481Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518747472278438:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.394194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518743177311081:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(406306023ns) {Configure:104(406298040ns) {ProcessConfigured:135(24340622ns) {StartInitialHandshake:157(24337466ns) {DropHandshakes:469(2821ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(496ns)} GenerateSessionVirtualId:988(767ns) PrepareNewSessionHandshake:476(3367ns) {SwitchToState:178(1573ns)}}}}}
2025-11-11T17:19:43.394583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.405749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518743177311079:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23269733ns) {IncomingHandshake:261(3508552ns) {DropIncomingHandshake:441(801ns) PrepareNewSessionHandshake:476(2614ns) {SwitchToState:178(1269ns)} GenerateSessionVirtualId:988(1665ns) IssueIncomingHandshakeReply:193(67429ns) {SwitchToState:178(507ns)}}}
2025-11-11T17:19:43.414052Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518747472278451:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:43.434049Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518747472278440:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.434073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518747472278440:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.441863Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.442654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518743177311079:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.442709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:43.442757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518743177311079:2048] [node 1] ICP22 created new session: [4:7571518747472278464:2048]
2025-11-11T17:19:43.442804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518747472278464:2048] [node 1] ICS09 handshake done sender: [4:7571518747472278451:2115] self: [4:7571518747472278459:0] peer: [1:7571518743861417797:0] socket: 348 qp: -1
2025-11-11T17:19:43.442836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518747472278464:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:43.442874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:43.443150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518743177311079:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:43.443419Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518747472278457:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.443441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518747472278457:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.446834Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518747472278457:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:43.453824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518747472278440:2105] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:43.489797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518747472278402:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.489859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:43.503743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518747472278402:2066] [node 5] ICP22 created new session: [4:7571518747472278470:2066]
2025-11-11T17:19:43.505746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518747472278410:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.506173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278469:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.517822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:43.517890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518747472278410:2062] [node 6] ICP22 created new session: [4:7571518747472278471:2062]
2025-11-11T17:19:43.537815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518747472278469:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.543970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518747472278470:2066] [node 5] ICS09 handshake done sender: [4:7571518747472278457:2117] self: [4:7571518747472278452:0] peer: [5:7571518747854247406:0] socket: 354 qp: -1
2025-11-11T17:19:43.550049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518747472278471:2062] [node 6] ICS09 handshake done sender: [4:7571518747472278440:2105] self: [4:7571518747472278439:0] peer: [6:7571518746838460363:0] socket: 346 qp: -1
2025-11-11T17:19:43.550104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518747472278471:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:43.550193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:43.550695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518747472278410:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(65164483ns) {HandleHandshakeStatus:329(65162533ns) {DropIncomingHandshake:441(1532ns) DropOutgoingHandshake:456(756ns) SwitchToState:178(4361ns) ProcessPendingSessionEvents:495(115028ns) {ForwardSessionEventToSession:612(67242ns) ForwardSessionEventToSession:612(3592ns) ForwardSessionEventToSession:612(5052ns) ForwardSessionEventToSession:612(3631ns) ForwardSessionEventToSession:612(5439ns) ForwardSessionEventToSession:612(5382ns) ForwardSessionEventToSession:612(2400ns) ForwardSessionEventToSession:612(1963ns) ForwardSessionEventToSession:612(4865ns) ForwardSessionEventToSession:612(3457ns)}}}
2025-11-11T17:19:43.550832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518747472278410:2062] [node 6] ICP18 handshake (actor [4:7571518747472278469:2120]) is too old
2025-11-11T17:19:43.036178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518749512171767:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:43.484612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518749848168171:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46887
2025-11-11T17:19:43.565830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460369:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.562043Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518748156385150:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.427600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518743559280040:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.484727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518749848168172:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46887
2025-11-11T17:19:43.561700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518747472278470:2066] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.605850Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.036219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:43.453991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518747854247397:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.561767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:43.598117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518743861417772:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.598200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:43.614801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518743861417773:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(61172341ns) {HandleHandshakeStatus:329(61167021ns) {DropIncomingHandshake:441(431ns) DropOutgoingHandshake:456(724ns) SwitchToState:178(554ns) ProcessPendingSessionEvents:495(28539892ns) {ForwardSessionEventToSession:612(16500023ns) ForwardSessionEventToSession:612(9729ns) ForwardSessionEventToSession:612(3181ns) ForwardSessionEventToSession:612(7777ns) ForwardSessionEventToSession:612(2741ns) ForwardSessionEventToSession:612(2401ns) ForwardSessionEventToSession:612(5055ns) ForwardSessionEventToSession:612(2302ns) ForwardSessionEventToSession:612(2578ns) ForwardSessionEventToSession:612(4549ns)}}}
2025-11-11T17:19:43.626092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518743861417772:2048] [node 2] ICP22 created new session: [1:7571518748156385169:2048]
2025-11-11T17:19:43.607788Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518746838460353:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:43.485082Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518749848168163:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.114059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518749512171757:2069] [node 5] ICP01 ready to work
2025-11-11T17:19:43.165709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [2:7571518745217204429:2061] [node 4] ICP35 event processing took too much time StateWork:293(27909636ns)
2025-11-11T17:19:43.221635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518745217204437:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.221684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518745217204437:2066] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.262898Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518745217204437:2066] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:43.335869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518745217204435:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.335948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:43.335999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518745217204435:2064] [node 6] ICP22 created new session: [2:7571518749512171791:2064]
2025-11-11T17:19:43.336043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518749512171791:2064] [node 6] ICS09 handshake done sender: [2:7571518745217204437:2066] self: [2:7571518745217204436:0] peer: [6:7571518746838460339:0] socket: 326 qp: -1
2025-11-11T17:19:43.336076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518749512171791:2064] [node 6] ICS10 traffic start
2025-11-11T17:19:43.336127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:43.384183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518749512171790:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5964
2025-11-11T17:19:43.397800Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518749512171790:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.398142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518745217204415:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(500047470ns) {Configure:104(500042560ns) {ProcessConfigured:135(16074105ns) {StartInitialHandshake:157(7093012ns) {DropHandshakes:469(3065ns) {DropIncomingHandshake:441(280ns) DropOutgoingHandshake:456(682ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:476(3026ns) {SwitchToState:178(728ns)}}}}}
2025-11-11T17:19:43.429937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518749512171757:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(263954726ns) {Configure:104(263949974ns) {ProcessConfigured:135(47757ns) {StartInitialHandshake:157(46211ns) {DropHandshakes:469(2418ns) {DropIncomingHandshake:441(475ns) DropOutgoingHandshake:456(688ns)} GenerateSessionVirtualId:988(397ns) PrepareNewSessionHandshake:476(936ns) {SwitchToState:178(430ns)}}}}}
2025-11-11T17:19:43.438004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518749512171802:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5964
2025-11-11T17:19:43.485938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518745217204414:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(605660598ns) {Configure:104(597286524ns) {ProcessConfigured:135(33558917ns) {StartInitialHandshake:157(29627174ns) {DropHandshakes:469(4835ns) {DropIncomingHandshake:441(1079ns) DropOutgoingHandshake:456(594ns)} GenerateSessionVirtualId:988(2886ns) PrepareNewSessionHandshake:476(2091ns) {SwitchToState:178(979ns)}}}}}
2025-11-11T17:19:43.493957Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518749512171802:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.525738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518745217204414:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.578405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518749512171805:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5964
2025-11-11T17:19:43.579985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518749512171806:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5964
2025-11-11T17:19:43.580441Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518749512171802:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:43.580656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518745217204414:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.580723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:43.580775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518745217204414:2048] [node 1] ICP22 created new session: [2:7571518749512171807:2048]
2025-11-11T17:19:43.580832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518749512171807:2048] [node 1] ICS09 handshake done sender: [2:7571518749512171802:2118] self: [2:7571518749512171804:0] peer: [1:7571518748156385149:0] socket: 368 qp: -1
2025-11-11T17:19:43.580868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518749512171807:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:43.580921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:43.629776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518749512171806:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.551556Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518749848168171:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.474150Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518747854247393:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.642148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518748156385169:2048] [node 2] ICS09 handshake done sender: [1:7571518748156385150:2116] self: [1:7571518748156385149:0] peer: [2:7571518749512171804:0] socket: 366 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.625873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518746838460345:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518746838460355:2116] held: no
2025-11-11T17:19:43.614346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278474:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.497852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518743559280027:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.551665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518749848168159:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.636219Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518746838460358:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:43.672136Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518749512171800:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.645952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518748156385169:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:43.651061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:43.671984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518748156385171:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4657
2025-11-11T17:19:43.694764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518748156385171:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.694884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518743861417773:2049] [node 3] ICP18 handshake (actor [1:7571518748156385171:2119]) is too old
2025-11-11T17:19:43.551833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518745553200793:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.690513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518747472278402:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(205134341ns) {HandleHandshakeStatus:329(205131301ns) {DropIncomingHandshake:441(14673ns) DropOutgoingHandshake:456(830ns) SwitchToState:178(719ns) ProcessPendingSessionEvents:495(17935608ns) {ForwardSessionEventToSession:612(17877254ns) ForwardSessionEventToSession:612(10442ns) ForwardSessionEventToSession:612(4036ns) ForwardSessionEventToSession:612(5915ns) ForwardSessionEventToSession:612(3079ns) ForwardSessionEventToSession:612(2875ns) ForwardSessionEventToSession:612(9197ns) ForwardSessionEventToSession:612(4562ns) ForwardSessionEventToSession:612(2231ns) ForwardSessionEventToSession:612(5148ns)}}}
2025-11-11T17:19:43.672174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518749512171800:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.638599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518742543492981:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.721645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518749512171795:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.721700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518749512171795:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.742071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518749512171795:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:43.509739Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.638652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:43.638700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518742543492981:2048] [node 1] ICP22 created new session: [6:7571518746838460372:2048]
2025-11-11T17:19:43.638766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518746838460372:2048] [node 1] ICS09 handshake done sender: [6:7571518746838460353:2114] self: [6:7571518746838460365:0] peer: [1:7571518748156385142:0] socket: 342 qp: -1
2025-11-11T17:19:43.638808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518746838460372:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:43.638848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:43.640561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460367:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.665986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460371:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.666234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518742543492983:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.666328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:43.666374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518742543492983:2050] [node 3] ICP22 created new session: [6:7571518746838460374:2050]
2025-11-11T17:19:43.666421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518746838460374:2050] [node 3] ICS09 handshake done sender: [6:7571518746838460358:2117] self: [6:7571518746838460366:0] peer: [3:7571518749848168162:0] socket: 355 qp: -1
2025-11-11T17:19:43.666454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518746838460374:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:43.666485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:43.671134Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518746838460371:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.671249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518742543492998:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.677912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518746838460345:2064] [node 4] ICP35 event processing took too much time PendingConnection:278(17557294ns) {HandleHandshakeStatus:395(17554134ns) {LogHandshakeFail:472(13502ns) {UpdateErrorStateLog:544(5058ns)} DropOutgoingHandshake:456(2011ns) ProcessPendingSessionEvents:495(10396139ns) {DropSessionEvent:512(13907ns) DropSessionEvent:512(5084ns) DropSessionEvent:512(5062ns) DropSessionEvent:512(5894ns) DropSessionEvent:512(4481ns) DropSessionEvent:512(9489665ns) DropSessionEvent:512(6049ns) DropSessionEvent:512(842655ns) DropSessionEvent:512(7997ns) DropSessionEvent:512(4712ns)}}}
2025-11-11T17:19:43.678033Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518746838460355:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:43.690095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518746838460377:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:568
2025-11-11T17:19:43.695104Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518746838460371:2123] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:43.698422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518746838460345:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.698464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:43.698505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518746838460345:2064] [node 4] ICP22 created new session: [6:7571518746838460378:2064]
2025-11-11T17:19:43.709923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518742543492998:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.710016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:43.710070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518742543492998:2060] [node 5] ICP22 created new session: [6:7571518746838460379:2060]
2025-11-11T17:19:43.710119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518746838460379:2060] [node 5] ICS09 handshake done sender: [6:7571518746838460371:2123] self: [6:7571518746838460376:0] peer: [5:7571518747854247403:0] socket: 384 qp: -1
2025-11-11T17:19:43.710170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518746838460379:2060] [node 5] ICS10 traffic start
2025-11-11T17:19:43.710205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:43.729971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518746838460378:2064] [node 4] ICS09 handshake done sender: [6:7571518746838460355:2116] self: [6:7571518746838460363:0] peer: [4:7571518747472278439:0] socket: 353 qp: -1
2025-11-11T17:19:43.730030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518746838460378:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:43.730088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:43.730321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518746838460345:2064] [node 4] ICP35 event processing took too much time PendingConnection:278(38868961ns) {HandleHandshakeStatus:329(31812110ns) {DropIncomingHandshake:441(343ns) DropOutgoingHandshake:456(618ns) SwitchToState:178(215ns) ProcessPendingSessionEvents:495(1199ns)}}
2025-11-11T17:19:43.573658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518749848168172:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.513790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518743559280025:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.513853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:43.513909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518743559280025:2048] [node 1] ICP22 created new session: [5:7571518747854247410:2048]
2025-11-11T17:19:43.513966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518747854247410:2048] [node 1] ICS09 handshake done sender: [5:7571518747854247393:2113] self: [5:7571518747854247401:0] peer: [1:7571518748156385140:0] socket: 329 qp: -1
2025-11-11T17:19:43.514008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518747854247410:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:43.514043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:43.521844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518743559280025:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:43.522366Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.543431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518743559280040:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22895) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518747854247398:2116] held: no
2025-11-11T17:19:43.543760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247409:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.566110Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518747854247398:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:43.581798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518743559280040:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.608175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247414:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.613770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247408:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.635573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:43.641110Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518747854247408:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.641206Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518747854247397:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:43.641339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518743559280026:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.642303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518743559280040:2061] [node 4] ICP22 created new session: [5:7571518747854247415:2061]
2025-11-11T17:19:43.646123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518743559280027:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.657864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518747854247415:2061] [node 4] ICS09 handshake done sender: [5:7571518747854247398:2116] self: [5:7571518747854247406:0] peer: [4:7571518747472278452:0] socket: 357 qp: -1
2025-11-11T17:19:43.657921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518747854247415:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:43.657986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:43.658273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518743559280040:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(76380790ns) {HandleHandshakeStatus:329(76378209ns) {DropIncomingHandshake:441(318ns) DropOutgoingHandshake:456(172ns) SwitchToState:178(597ns) ProcessPendingSessionEvents:495(1414ns)}}
2025-11-11T17:19:43.677808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:43.679056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518743559280027:2050] [node 3] ICP22 created new session: [5:7571518747854247418:2050]
2025-11-11T17:19:43.681758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518747854247404:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.681784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518747854247404:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.691219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518747854247418:2050] [node 3] ICS09 handshake done sender: [5:7571518747854247397:2115] self: [5:7571518747854247407:0] peer: [3:7571518749848168157:0] socket: 356 qp: -1
2025-11-11T17:19:43.694542Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518747854247404:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:43.695350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518747854247384:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.698345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518747854247418:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:43.733730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:43.753711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:43.758023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518747854247384:2069] [node 6] ICP22 created new session: [5:7571518747854247421:2069]
2025-11-11T17:19:43.765713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518743861417772:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(115161878ns) {HandleHandshakeStatus:329(115158788ns) {DropIncomingHandshake:441(21018ns) DropOutgoingHandshake:456(580ns) SwitchToState:178(624ns) ProcessPendingSessionEvents:495(18406137ns) {ForwardSessionEventToSession:612(10741700ns) ForwardSessionEventToSession:612(9632ns) ForwardSessionEventToSession:612(2890ns) ForwardSessionEventToSession:612(5544ns) ForwardSessionEventToSession:612(8497ns) ForwardSessionEventToSession:612(2895ns) ForwardSessionEventToSession:612(5156ns) ForwardSessionEventToSession:612(2629ns) ForwardSessionEventToSession:612(3503ns) ForwardSessionEventToSession:612(7606617ns)}}}
2025-11-11T17:19:43.586240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518749848168158:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.586278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518749848168158:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.590415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518745553200810:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.590524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:43.590595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518745553200810:2062] [node 6] ICP22 created new session: [3:7571518749848168175:2062]
2025-11-11T17:19:43.590644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518749848168175:2062] [node 6] ICS09 handshake done sender: [3:7571518749848168163:2113] self: [3:7571518749848168162:0] peer: [6:7571518746838460366:0] socket: 324 qp: -1
2025-11-11T17:19:43.590692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518749848168175:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:43.590755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:43.619103Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518749848168158:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:43.642063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518749848168170:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46887
2025-11-11T17:19:43.659509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518745553200809:2059] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.659585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:43.659645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518745553200809:2059] [node 5] ICP22 created new session: [3:7571518749848168182:2059]
2025-11-11T17:19:43.659706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518749848168182:2059] [node 5] ICS09 handshake done sender: [3:7571518749848168158:2111] self: [3:7571518749848168157:0] peer: [5:7571518747854247407:0] socket: 351 qp: -1
2025-11-11T17:19:43.659737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518749848168182:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:43.659791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:43.661991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518749848168181:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46887
2025-11-11T17:19:43.670217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518749848168180:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46887
2025-11-11T17:19:43.670417Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518749848168170:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.670892Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518749848168171:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:43.671602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518745553200794:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.693756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518745553200793:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.701892Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518749848168166:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.701918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518749848168166:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:43.702525Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.703918Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518749848168166:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:43.713853Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:43.714048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:43.714097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518745553200793:2048] [node 1] ICP22 created new session: [3:7571518749848168187:2048]
2025-11-11T17:19:43.714139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518749848168187:2048] [node 1] ICS09 handshake done sender: [3:7571518749848168171:2117] self: [3:7571518749848168173:0] peer: [1:7571518743861417809:0] socket: 379 qp: -1
2025-11-11T17:19:43.714175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518749848168187:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:43.714211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:43.714630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518745553200793:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20712866ns) {HandleHandshakeStatus:329(20708904ns) {DropIncomingHandshake:441(3501ns) DropOutgoingHandshake:456(11214ns) SwitchToState:178(386ns) ProcessPendingSessionEvents:495(91552ns) {ForwardSessionEventToSession:612(47095ns) ForwardSessionEventToSession:612(2192ns) ForwardSessionEventToSession:612(5626ns) ForwardSessionEventToSession:612(2435ns) ForwardSessionEventToSession:612(5184ns) ForwardSessionEventToSession:612(5336ns) ForwardSessionEventToSession:612(2387ns) ForwardSessionEventToSession:612(7346ns) ForwardSessionEventToSession:612(2240ns) ForwardSessionEventToSession:612(2124ns)}}}
2025-11-11T17:19:43.714701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518745553200810:2062] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:43.714762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518745553200793:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:43.734148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518745553200822:2072] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.734225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:43.734271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518745553200822:2072] [node 4] ICP22 created new session: [3:7571518749848168190:2072]
2025-11-11T17:19:43.734320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518749848168190:2072] [node 4] ICS09 handshake done sender: [3:7571518749848168166:2114] self: [3:7571518749848168165:0] peer: [4:7571518747472278475:0] socket: 386 qp: -1
2025-11-11T17:19:43.734351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518749848168190:2072] [node 4] ICS10 traffic start
2025-11-11T17:19:43.734406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:43.735233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518749848168189:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46887
2025-11-11T17:19:43.735599Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518749848168170:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:43.735770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518745553200794:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.735834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:43.735884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518745553200794:2049] [node 2] ICP22 created new session: [3:7571518749848168192:2049]
2025-11-11T17:19:43.735923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518749848168192:2049] [node 2] ICS09 handshake done sender: [3:7571518749848168170:2116] self: [3:7571518749848168185:0] peer: [2:7571518749512171793:0] socket: 365 qp: -1
2025-11-11T17:19:43.735952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518749848168192:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:43.735993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:43.694397Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518747472278474:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:43.774035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518747854247421:2069] [node 6] ICS09 handshake done sender: [5:7571518747854247404:2119] self: [5:7571518747854247403:0] peer: [6:7571518746838460376:0] socket: 382 qp: -1
2025-11-11T17:19:43.694541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518743177311081:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:43.703838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518747472278476:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28505
2025-11-11T17:19:43.742029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518747472278474:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:43.761632Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10167) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.769848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518743177311081:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10167) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518747472278474:2121] held: no
2025-11-11T17:19:43.779761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518743177311081:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.779814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:43.773822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518749512171800:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:43.783063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518747854247421:2069] [node 6] ICS10 traffic start
2025-11-11T17:19:43.782132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518749512171757:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.786394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518747854247422:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25208
2025-11-11T17:19:43.793827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:43.794228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518747854247384:2069] [node 6] ICP35 event processing took too much time PendingConnection:278(98694043ns) {HandleHandshakeStatus:329(98691983ns) {DropIncomingHandshake:441(17708ns) DropOutgoingHandshake:456(764ns) SwitchToState:178(350ns) ProcessPendingSessionEvents:495(87791ns) {ForwardSessionEventToSession:612(44255ns) ForwardSessionEventToSession:612(3433ns) ForwardSessionEventToSession:612(4973ns) ForwardSessionEventToSession:612(3315ns) ForwardSessionEventToSession:612(4862ns) ForwardSessionEventToSession:612(7535ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(5413ns) ForwardSessionEventToSession:612(1772ns) ForwardSessionEventToSession:612(1762ns)}}}
2025-11-11T17:19:43.798297Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518747854247408:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:43.805695Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:43.806233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518743559280026:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518747854247408:2120] held: no
2025-11-11T17:19:43.814349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518743559280026:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:43.814432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:43.814475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518743559280026:2049] [node 2] ICP22 created new session: [5:7571518747854247424:2049]
2025-11-11T17:19:43.814514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518747854247424:2049] [node 2] ICS09 handshake done sender: [5:7571518747854247408:2120] self: [5:7571518747854247416:0] peer: [2:7571518749512171799:0] socket: 377 qp: -1
2025-11-11T17:19:43.814546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518747854247424:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:43.814572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:43.782226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:43.782283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518749512171757:2069] [node 5] ICP22 created new session: [2:7571518749512171812:2069]
2025-11-11T17:19:43.782334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518749512171812:2069] [node 5] ICS09 handshake done sender: [2:7571518749512171800:2117] self: [2:7571518749512171799:0] peer: [5:7571518747854247416:0] socket: 369 qp: -1
2025-11-11T17:19:43.782371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518749512171812:2069] [node 5] ICS10 traffic start
2025-11-11T17:19:43.782427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:43.789659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518745217204415:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:43.801713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:43.801769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518745217204415:2049] [node 3] ICP22 created new session: [2:7571518749512171814:2049]
2025-11-11T17:19:43.813677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518749512171814:2049] [node 3] ICS09 handshake done sender: [2:7571518749512171795:2115] self: [2:7571518749512171793:0] peer: [3:7571518749848168185:0] socket: 364 qp: -1
2025-11-11T17:19:43.813742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518749512171814:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:43.813801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:43.818942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518745217204415:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(36284960ns) {HandleHandshakeStatus:329(32356464ns) {DropIncomingHandshake:441(17356ns) DropOutgoingHandshake:456(625ns) SwitchToState:178(656ns) ProcessPendingSessionEvents:495(109244ns) {ForwardSessionEventToSession:612(60131ns) ForwardSessionEventToSession:612(2953ns) ForwardSessionEventToSession:612(10646ns) ForwardSessionEventToSession:612(2826ns) ForwardSessionEventToSession:612(4782ns) ForwardSessionEventToSession:612(4775ns) ForwardSessionEventToSession:612(3097ns) ForwardSessionEventToSession:612(5073ns) ForwardSessionEventToSession:612(2522ns) ForwardSessionEventToSession:612(3034ns)}}}
2025-11-11T17:19:43.793662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518743177311081:2050] [node 3] ICP22 created new session: [4:7571518747472278477:2050]
2025-11-11T17:19:43.801691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518747472278477:2050] [node 3] ICS09 handshake done sender: [4:7571518747472278474:2121] self: [4:7571518747472278475:0] peer: [3:7571518749848168165:0] socket: 387 qp: -1
2025-11-11T17:19:43.805669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518747472278477:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:43.813718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:43.813991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518743177311081:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(34159478ns) {HandleHandshakeStatus:329(34156560ns) {DropIncomingHandshake:441(834ns) DropOutgoingHandshake:456(563ns) SwitchToState:178(627ns) ProcessPendingSessionEvents:495(1093ns)}}
2025-11-11T17:19:43.821750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518743559280027:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(151602362ns) {HandleHandshakeStatus:329(151599734ns) {DropIncomingHandshake:441(4173ns) DropOutgoingHandshake:456(26336ns) SwitchToState:178(711ns) ProcessPendingSessionEvents:495(25754938ns) {ForwardSessionEventToSession:612(7664643ns) ForwardSessionEventToSession:612(16965ns) ForwardSessionEventToSession:612(6008661ns) ForwardSessionEventToSession:612(3839ns) ForwardSessionEventToSession:612(2868ns) ForwardSessionEventToSession:612(4504145ns) ForwardSessionEventToSession:612(9074ns) ForwardSessionEventToSession:612(7516359ns) ForwardSessionEventToSession:612(3626ns) ForwardSessionEventToSession:612(2968ns)}}}
2025-11-11T17:19:43.910030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:43.910284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518749512171807:2048] [node 1] ICS01 socket: 368 reason# EndOfStream
2025-11-11T17:19:43.910334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518745217204414:2048] [node 1] ICP30 unregister session Session# [2:7571518749512171807:2048] VirtualId# [2:7571518749512171804:0]
2025-11-11T17:19:43.910373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518749512171807:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.910501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:43.910555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:43.913776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:43.914018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518749848168187:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-11-11T17:19:43.914056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518745553200793:2048] [node 1] ICP30 unregister session Session# [3:7571518749848168187:2048] VirtualId# [3:7571518749848168173:0]
2025-11-11T17:19:43.914092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518749848168187:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.914201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:43.914246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:43.921636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:43.921852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518746838460372:2048] [node 1] ICS01 socket: 342 reason# EndOfStream
2025-11-11T17:19:43.921895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518742543492981:2048] [node 1] ICP30 unregister session Session# [6:7571518746838460372:2048] VirtualId# [6:7571518746838460365:0]
2025-11-11T17:19:43.921928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518746838460372:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.922030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:43.922067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:43.926891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:43.927138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518747854247410:2048] [node 1] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-11T17:19:43.927170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518747854247410:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.927248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:43.927282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518747854247410:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:43.928160Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Couldn't connect to any resolved address
2025-11-11T17:19:43.928389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518743559280025:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:43.928420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518747854247410:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:43.928443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518743559280025:2048] [node 1] ICP30 unregister session Session# [5:7571518747854247410:2048] VirtualId# [5:7571518747854247401:0]
2025-11-11T17:19:43.928478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:43.928496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518743559280025:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:43.928505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12562) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:43.925706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:43.925945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518747472278464:2048] [node 1] ICS01 socket: 348 reason# EndOfStream
2025-11-11T17:19:43.925982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518743177311079:2048] [node 1] ICP30 unregister session Session# [4:7571518747472278464:2048] VirtualId# [4:7571518747472278459:0]
2025-11-11T17:19:43.926017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518747472278464:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.926114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:43.926155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:43.942901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518743559280025:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:43.983005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:43.983260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518749848168192:2049] [node 2] ICS01 socket: 365 reason# EndOfStream
2025-11-11T17:19:43.983306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518745553200794:2049] [node 2] ICP30 unregister session Session# [3:7571518749848168192:2049] VirtualId# [3:7571518749848168185:0]
2025-11-11T17:19:43.983340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518749848168192:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.983428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:43.983474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:43.993728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:43.993961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518746838460356:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-11T17:19:43.994006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518742543492982:2049] [node 2] ICP30 unregister session Session# [6:7571518746838460356:2049] VirtualId# [6:7571518746838460339:0]
2025-11-11T17:19:43.994043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518746838460356:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.994159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:43.994206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:43.989791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:44.002980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518747472278446:2049] [node 2] ICS01 socket: 334 reason# EndOfStream
2025-11-11T17:19:43.989444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:43.989691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518747854247424:2049] [node 2] ICS01 socket: 377 reason# EndOfStream
2025-11-11T17:19:43.989735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518743559280026:2049] [node 2] ICP30 unregister session Session# [5:7571518747854247424:2049] VirtualId# [5:7571518747854247416:0]
2025-11-11T17:19:43.989786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518747854247424:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:43.989868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:44.003033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:44.006426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518743177311080:2049] [node 2] ICP30 unregister session Session# [4:7571518747472278446:2049] VirtualId# [4:7571518747472278404:0]
2025-11-11T17:19:44.006486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518747472278446:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:44.015570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:44.016005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:44.137773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:44.137994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518747854247418:2050] [node 3] ICS07 socket disconnect 356 reason# EndOfStream
2025-11-11T17:19:44.138022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518747854247418:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:44.138129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:44.138172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518747854247418:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:44.138956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:44.139149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518746838460374:2050] [node 3] ICS01 socket: 355 reason# EndOfStream
2025-11-11T17:19:44.139187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518742543492983:2050] [node 3] ICP30 unregister session Session# [6:7571518746838460374:2050] VirtualId# [6:7571518746838460366:0]
2025-11-11T17:19:44.139217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518746838460374:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:44.139295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:44.139339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:44.139428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:44.139598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518747472278477:2050] [node 3] ICS01 socket: 387 reason# EndOfStream
2025-11-11T17:19:44.139643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518743177311081:2050] [node 3] ICP30 unregister session Session# [4:7571518747472278477:2050] VirtualId# [4:7571518747472278475:0]
2025-11-11T17:19:44.139668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518747472278477:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:44.139727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:44.139764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:44.141439Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10167) Couldn't connect to any resolved address
2025-11-11T17:19:44.141608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518743559280027:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10167) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:44.141663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518747854247418:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:44.153889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518743559280027:2050] [node 3] ICP30 unregister session Session# [5:7571518747854247418:2050] VirtualId# [5:7571518747854247407:0]
2025-11-11T17:19:44.167046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:44.167119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518743559280027:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:44.167135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:44.167399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [5:7571518743559280027:2050] [node 3] ICP35 event processing took too much time StateWork:293(25612650ns) {HandleHandshakeStatus:395(25609095ns) {LogHandshakeFail:472(15325ns) {UpdateErrorStateLog:544(1592ns)} DropOutgoingHandshake:456(570ns) ProcessPendingSessionEvents:495(78ns) UnregisterSession:538(24742682ns) {ProcessPendingSessionEvents:495(2234ns) SwitchToInitialState:201(7883ns) {SwitchToState:178(4418ns)}} TransitToErrorState:752(62939ns) {SwitchToState:178(6002ns) ProcessPendingSessionEvents:495(632ns) DropHandshakes:469(2377ns) {DropIncomingHandshake:441(414ns) DropOutgoingHandshake:456(599ns)}}}}
2025-11-11T17:19:44.174155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518743559280027:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:44.266549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:44.266722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518747854247415:2061] [node 4] ICS01 socket: 357 reason# EndOfStream
2025-11-11T17:19:44.265976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:44.266754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518743559280040:2061] [node 4] ICP30 unregister session Session# [5:7571518747854247415:2061] VirtualId# [5:7571518747854247406:0]
2025-11-11T17:19:44.266779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518747854247415:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:44.266226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518746838460378:2064] [node 4] ICS01 socket: 353 reason# EndOfStream
2025-11-11T17:19:44.266268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518746838460345:2064] [node 4] ICP30 unregister session Session# [6:7571518746838460378:2064] VirtualId# [6:7571518746838460363:0]
2025-11-11T17:19:44.266830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:44.266862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:44.266301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518746838460378:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:44.266389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:44.266435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:44.372341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:44.372531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518746838460379:2060] [node 5] ICS01 socket: 384 reason# EndOfStream
2025-11-11T17:19:44.372557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518742543492998:2060] [node 5] ICP30 unregister session Session# [6:7571518746838460379:2060] VirtualId# [6:7571518746838460376:0]
2025-11-11T17:19:44.372589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518746838460379:2060] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:44.372697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:44.372733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:44.613973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518751439575692:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:44.614021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518751439575693:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:44.713979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518752710266325:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:44.714028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518752710266326:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:44.755448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518751058712057:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:44.755491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518751058712058:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:44.797796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518752748538098:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:44.797842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518752748538099:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:44.797872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518752748538100:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:44.877897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518752125797340:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:44.877940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518752125797341:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:44.877967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518752125797342:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:44.973878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518751422732651:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:44.973917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518751422732652:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:44.973942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518751422732653:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:45.355700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518756420764666:2078] [node 4] ICP01 ready to work
2025-11-11T17:19:45.358376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518755734543049:2107] [node 4] ICP01 ready to work
2025-11-11T17:19:45.357506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518755717699965:2064] [node 5] ICP01 ready to work
2025-11-11T17:19:45.376783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518755353679400:2093] [node 4] ICP01 ready to work
2025-11-11T17:19:45.385151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518755734543053:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.464451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518757005233683:2096] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.522908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518757043505450:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:45.450412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518756420764678:2082] [node 6] ICP01 ready to work
2025-11-11T17:19:45.472850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518755734543049:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(57045209ns) {Configure:104(57035043ns) {ProcessConfigured:135(68234ns) {StartInitialHandshake:157(65360ns) {DropHandshakes:469(3053ns) {DropIncomingHandshake:441(625ns) DropOutgoingHandshake:456(615ns)} GenerateSessionVirtualId:988(882ns) PrepareNewSessionHandshake:476(2044ns) {SwitchToState:178(535ns)}}}}}
2025-11-11T17:19:45.509656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518755734543052:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-11T17:19:45.557993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518755353679412:2095] [node 6] ICP01 ready to work
2025-11-11T17:19:45.536183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518757043505446:2089] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.577627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518755734543060:2108] [node 5] ICP01 ready to work
2025-11-11T17:19:45.612387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518755734543053:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(226789177ns) {Configure:104(226784098ns) {ProcessConfigured:135(20387369ns) {StartInitialHandshake:157(20383823ns) {DropHandshakes:469(2908ns) {DropIncomingHandshake:441(459ns) DropOutgoingHandshake:456(301ns)} GenerateSessionVirtualId:988(1058ns) PrepareNewSessionHandshake:476(5072ns) {SwitchToState:178(2691ns)}}}}}
2025-11-11T17:19:45.628625Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543052:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.638461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518755734543064:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-11T17:19:45.625625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518757005233697:2097] [node 5] ICP01 ready to work
2025-11-11T17:19:45.601841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518757043505460:2083] [node 5] ICP01 ready to work
2025-11-11T17:19:45.628346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518757043505465:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:45.634708Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518757043505450:2100] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.606237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518755353679400:2093] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(229160482ns) {Configure:104(229156951ns) {ProcessConfigured:135(76214ns) {StartInitialHandshake:157(73103ns) {DropHandshakes:469(2546ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(493ns)} GenerateSessionVirtualId:988(709ns) PrepareNewSessionHandshake:476(1573ns) {SwitchToState:178(739ns)}}}}}
2025-11-11T17:19:45.661671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518755717699992:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.663383Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518751439575693:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(304715496ns) {Configure:104(304712578ns) {ProcessConfigured:135(29441197ns) {StartInitialHandshake:157(29438587ns) {DropHandshakes:469(14339841ns) {DropIncomingHandshake:441(578ns) DropOutgoingHandshake:456(1379ns)} GenerateSessionVirtualId:988(2606ns) PrepareNewSessionHandshake:476(3579ns) {SwitchToState:178(1965ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.721737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518752748538098:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.605832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518755353679420:2097] [node 5] ICP01 ready to work
2025-11-11T17:19:45.630962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518757005233696:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18243
2025-11-11T17:19:45.722349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518756420764666:2078] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(366023865ns) {Configure:104(366014650ns) {ProcessConfigured:135(20161095ns) {StartInitialHandshake:157(20158324ns) {DropHandshakes:469(1414ns) {DropIncomingHandshake:441(324ns) DropOutgoingHandshake:456(278ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(3065ns) {SwitchToState:178(1300ns)}}}}}
2025-11-11T17:19:45.722905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518756420764706:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:45.718417Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543055:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.750303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518752125797342:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(394308336ns) {Configure:104(394303697ns) {ProcessConfigured:135(67326ns) {StartInitialHandshake:157(63625ns) {DropHandshakes:469(2494ns) {DropIncomingHandshake:441(836ns) DropOutgoingHandshake:456(340ns)} GenerateSessionVirtualId:988(898ns) PrepareNewSessionHandshake:476(1345ns) {SwitchToState:178(512ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.703982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518755717699965:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(346170974ns) {Configure:104(346167002ns) {ProcessConfigured:135(71773ns) {StartInitialHandshake:157(68995ns) {DropHandshakes:469(1727ns) {DropIncomingHandshake:441(435ns) DropOutgoingHandshake:456(310ns)} GenerateSessionVirtualId:988(430ns) PrepareNewSessionHandshake:476(1773ns) {SwitchToState:178(919ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.631994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518757005233698:2098] [node 6] ICP01 ready to work
2025-11-11T17:19:45.746010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518752748538098:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(47228924ns) {IncomingHandshake:261(47226693ns) {DropIncomingHandshake:441(498ns) PrepareNewSessionHandshake:476(2489ns) {SwitchToState:178(1141ns)} GenerateSessionVirtualId:988(936ns) IssueIncomingHandshakeReply:193(35107021ns) {SwitchToState:178(3917ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.780093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518757043505468:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:45.780155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518757043505469:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:45.713912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518755353679412:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(107373247ns) {Configure:104(107367247ns) {ProcessConfigured:135(13821648ns) {StartInitialHandshake:157(13814728ns) {DropHandshakes:469(2462ns) {DropIncomingHandshake:441(543ns) DropOutgoingHandshake:456(508ns)} GenerateSessionVirtualId:988(2721ns) PrepareNewSessionHandshake:476(4607ns) {SwitchToState:178(2068ns)}}}}}
2025-11-11T17:19:45.704676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700015:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.817696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518756420764680:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.817745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518756420764680:2091] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.718457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518755734543055:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:45.818771Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518756420764680:2091] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.704887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700016:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.677907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233696:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.757430Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518755734543055:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:45.716019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518755353679430:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5678
2025-11-11T17:19:45.819063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518756420764678:2082] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.751719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518757005233699:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18243
2025-11-11T17:19:45.789833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518757043505464:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.764329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518755353679432:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5678
2025-11-11T17:19:45.705002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700017:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.726763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518751422732653:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(283579267ns) {Configure:104(283574324ns) {ProcessConfigured:135(67944768ns) {StartInitialHandshake:157(67941955ns) {DropHandshakes:469(6099ns) {DropIncomingHandshake:441(4116ns) DropOutgoingHandshake:456(343ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:476(4262ns) {SwitchToState:178(1818ns)}}}}}
2025-11-11T17:19:45.729669Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518755717700016:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.756884Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518755717700017:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.757221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700023:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.757825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518751422732652:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(177534679ns) {Configure:104(177529242ns) {ProcessConfigured:135(55719ns) {StartInitialHandshake:157(54188ns) {DropHandshakes:469(2276ns) {DropIncomingHandshake:441(461ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:476(1466ns) {SwitchToState:178(622ns)}}}}}
2025-11-11T17:19:45.763191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518755717699992:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(76903346ns) {Configure:104(76897943ns) {ProcessConfigured:135(69286ns) {StartInitialHandshake:157(67378ns) {DropHandshakes:469(2893ns) {DropIncomingHandshake:441(752ns) DropOutgoingHandshake:456(437ns)} GenerateSessionVirtualId:988(697ns) PrepareNewSessionHandshake:476(1923ns) {SwitchToState:178(499ns)}}}}}
2025-11-11T17:19:45.763393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518755717699992:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.768374Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518755717700015:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.805824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518755717699965:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.838495Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518755717700023:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.838647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518751422732653:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.842224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700030:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.842299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700031:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.842356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700032:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.906538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518751422732651:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(456439060ns) {Configure:104(456431957ns) {ProcessConfigured:135(38440442ns) {StartInitialHandshake:157(38437760ns) {DropHandshakes:469(2437ns) {DropIncomingHandshake:441(411ns) DropOutgoingHandshake:456(487ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:476(2701ns) {SwitchToState:178(823ns)}}}}}
2025-11-11T17:19:45.906781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518751422732651:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.776755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518757005233698:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(102354749ns) {Configure:104(102349157ns) {ProcessConfigured:135(80789ns) {StartInitialHandshake:157(78258ns) {DropHandshakes:469(2108ns) {DropIncomingHandshake:441(458ns) DropOutgoingHandshake:456(524ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:476(1523ns) {SwitchToState:178(465ns)}}}}}
2025-11-11T17:19:45.773849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518755734543049:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(19995077ns) {IncomingHandshake:261(54479ns) {DropIncomingHandshake:441(620ns) PrepareNewSessionHandshake:476(1930ns) {SwitchToState:178(924ns)} GenerateSessionVirtualId:988(1098ns) IssueIncomingHandshakeReply:193(6837ns)}}
2025-11-11T17:19:45.785900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518752710266325:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(316013151ns) {Configure:104(308043762ns) {ProcessConfigured:135(12009422ns) {StartInitialHandshake:157(12005933ns) {DropHandshakes:469(2120ns) {DropIncomingHandshake:441(413ns) DropOutgoingHandshake:456(466ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:476(3931ns) {SwitchToState:178(1620ns)}}}}}
2025-11-11T17:19:45.786028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518752710266325:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.830875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518757005233683:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(355782791ns) {Configure:104(355779016ns) {ProcessConfigured:135(31766887ns) {StartInitialHandshake:157(31764505ns) {DropHandshakes:469(1658ns) {DropIncomingHandshake:441(390ns) DropOutgoingHandshake:456(330ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:476(3432ns) {SwitchToState:178(1531ns)}}}}}
2025-11-11T17:19:45.831586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518757005233709:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18243
2025-11-11T17:19:45.841014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233699:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.890467Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233670:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.890512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518757005233670:2095] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:45.891131Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233709:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.895553Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518757005233670:2095] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:45.895790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518757005233712:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18243
2025-11-11T17:19:45.921881Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:45.789896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518757043505464:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:45.819103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:45.782230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518755353679417:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5678
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.866066Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543064:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.819166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518756420764678:2082] [node 6] ICP22 created new session: [5:7571518756420764714:2082]
2025-11-11T17:19:45.849914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518752748538100:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(443948219ns) {Configure:104(443941762ns) {ProcessConfigured:135(48038169ns) {StartInitialHandshake:157(48035716ns) {DropHandshakes:469(1874ns) {DropIncomingHandshake:441(344ns) DropOutgoingHandshake:456(364ns)} GenerateSessionVirtualId:988(727ns) PrepareNewSessionHandshake:476(3741ns) {SwitchToState:178(2066ns)}}}}}
2025-11-11T17:19:45.839185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518751058712058:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(364963459ns) {Configure:104(364958545ns) {ProcessConfigured:135(27933254ns) {StartInitialHandshake:157(27931130ns) {DropHandshakes:469(2031ns) {DropIncomingHandshake:441(371ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:476(2207ns) {SwitchToState:178(180ns)}}}}}
2025-11-11T17:19:45.934460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700037:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
2025-11-11T17:19:45.866222Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543057:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.934851Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518755717700017:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:45.841257Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679427:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.841281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518755353679427:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:45.846267Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679430:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.856391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518755353679436:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5678
2025-11-11T17:19:45.862725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518755353679427:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:45.867194Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679417:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.867311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518751058712058:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.878220Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679432:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.938005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679422:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.938036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518755353679422:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:45.941724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518755353679420:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(30247583ns) {IncomingHandshake:261(15975359ns) {DropIncomingHandshake:441(210ns) PrepareNewSessionHandshake:476(2080ns) {SwitchToState:178(1226ns)} GenerateSessionVirtualId:988(397ns) IssueIncomingHandshakeReply:193(12880ns)}}
2025-11-11T17:19:45.949857Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518755353679422:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:45.957704Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679436:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.957784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518755353679412:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:45.957870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:45.967112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518755353679412:2095] [node 6] ICP22 created new session: [3:7571518755353679442:2095]
2025-11-11T17:19:45.819219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518756420764714:2082] [node 6] ICS09 handshake done sender: [5:7571518756420764680:2091] self: [5:7571518756420764679:0] peer: [6:7571518755717700028:0] socket: 333 qp: -1
2025-11-11T17:19:45.881790Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518757043505464:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:45.977913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518755353679442:2095] [node 6] ICS09 handshake done sender: [3:7571518755353679427:2113] self: [3:7571518755353679426:0] peer: [6:7571518755717700029:0] socket: 353 qp: -1
2025-11-11T17:19:45.994479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518755353679400:2093] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:45.994543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:45.994588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518755353679400:2093] [node 4] ICP22 created new session: [3:7571518755353679444:2093]
2025-11-11T17:19:45.994636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518755353679444:2093] [node 4] ICS09 handshake done sender: [3:7571518755353679422:2111] self: [3:7571518755353679421:0] peer: [4:7571518757043505481:0] socket: 341 qp: -1
2025-11-11T17:19:45.994670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518755353679444:2093] [node 4] ICS10 traffic start
2025-11-11T17:19:45.994717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:45.995009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518755353679400:2093] [node 4] ICP18 handshake (actor [3:7571518755353679436:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.965961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518757005233697:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(300117464ns) {Configure:104(300110823ns) {ProcessConfigured:135(77465ns) {StartInitialHandshake:157(75424ns) {DropHandshakes:469(1389ns) {DropIncomingHandshake:441(404ns) DropOutgoingHandshake:456(261ns)} GenerateSessionVirtualId:988(500ns) PrepareNewSessionHandshake:476(1393ns) {SwitchToState:178(500ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.819262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518756420764714:2082] [node 6] ICS10 traffic start
2025-11-11T17:19:45.866246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518755734543057:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.943461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518755717700036:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.997738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518755353679442:2095] [node 6] ICS10 traffic start
2025-11-11T17:19:45.882430Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518757043505450:2100] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:45.966027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518752710266326:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:45.966057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:45.966100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518752710266326:2049] [node 3] ICP22 created new session: [2:7571518757005233715:2049]
2025-11-11T17:19:45.966137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518757005233715:2049] [node 3] ICS09 handshake done sender: [2:7571518757005233670:2095] self: [2:7571518757005233669:0] peer: [3:7571518755353679438:0] socket: 332 qp: -1
2025-11-11T17:19:45.966172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518757005233715:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:45.966198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:45.966736Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518757005233696:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:46.005901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518752710266325:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.014639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518757005233718:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18243
2025-11-11T17:19:46.017735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:46.017783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518752710266325:2048] [node 1] ICP22 created new session: [2:7571518761300201016:2048]
2025-11-11T17:19:46.021065Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233718:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.882836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518755734543071:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-11T17:19:45.819298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.886217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518755734543049:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.037821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518761300201016:2048] [node 1] ICS09 handshake done sender: [2:7571518757005233696:2119] self: [2:7571518757005233705:0] peer: [1:7571518755734543056:0] socket: 337 qp: -1
2025-11-11T17:19:46.046025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518761300201016:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:45.944574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518755717699965:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518755717700015:2111] held: no
2025-11-11T17:19:46.004236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:45.894001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518752748538098:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:45.830042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518752125797341:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(359590736ns) {Configure:104(359582307ns) {ProcessConfigured:135(30941305ns) {StartInitialHandshake:157(30938707ns) {DropHandshakes:469(2690ns) {DropIncomingHandshake:441(513ns) DropOutgoingHandshake:456(437ns)} GenerateSessionVirtualId:988(939ns) PrepareNewSessionHandshake:476(5292ns) {SwitchToState:178(2354ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.894064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:45.886283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:45.833925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518752125797340:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(419900242ns) {Configure:104(419895479ns) {ProcessConfigured:135(31962008ns) {StartInitialHandshake:157(31960036ns) {DropHandshakes:469(849ns) {DropIncomingHandshake:441(268ns) DropOutgoingHandshake:456(253ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(7313ns) {SwitchToState:178(1290ns)}}}}}
2025-11-11T17:19:45.850515Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518756420764706:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.881922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518756420764678:2082] [node 6] ICP18 handshake (actor [5:7571518756420764706:2112]) is too old
2025-11-11T17:19:45.996341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518756420764719:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.018440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518760715732016:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.037913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518756420764719:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.039803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518760715732017:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.039881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518760715732018:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.054845Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518760715732016:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.055018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518760715732017:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.061818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518752125797340:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:46.069853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518752125797342:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:46.078507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518752125797340:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(21283507ns) {IncomingHandshake:261(21281814ns) {DropIncomingHandshake:441(782ns) PrepareNewSessionHandshake:476(1320ns) {SwitchToState:178(559ns)} GenerateSessionVirtualId:988(386ns) IssueIncomingHandshakeReply:193(10870474ns) {SwitchToState:178(2217ns)}}}
2025-11-11T17:19:45.944633Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518755717700015:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:46.012458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518755353679443:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5678
2025-11-11T17:19:46.058314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:45.894121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518752748538098:2048] [node 1] ICP22 created new session: [4:7571518757043505474:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.945759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518755717699992:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:45.886331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518755734543049:2107] [node 4] ICP22 created new session: [1:7571518755734543072:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:46.074001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518761300201020:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18243
2025-11-11T17:19:46.096097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518752125797341:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:46.102770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518760715732022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.015605Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679443:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.894174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518757043505474:2048] [node 1] ICS09 handshake done sender: [4:7571518757043505450:2100] self: [4:7571518757043505467:0] peer: [1:7571518755734543054:0] socket: 338 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.886375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518755734543072:2107] [node 4] ICS09 handshake done sender: [1:7571518755734543055:2111] self: [1:7571518755734543054:0] peer: [4:7571518757043505467:0] socket: 335 qp: -1
2025-11-11T17:19:46.074109Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:46.033724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518751058712057:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:46.070226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518755353679441:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5678
2025-11-11T17:19:46.070630Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518755353679417:2109] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:46.070793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518751058712058:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.070859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:46.070892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518751058712058:2049] [node 2] ICP22 created new session: [3:7571518759648646746:2049]
2025-11-11T17:19:46.070938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518759648646746:2049] [node 2] ICS09 handshake done sender: [3:7571518755353679417:2109] self: [3:7571518755353679438:0] peer: [2:7571518757005233669:0] socket: 334 qp: -1
2025-11-11T17:19:46.070965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518759648646746:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:46.070991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:46.098429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518759648646748:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5678
2025-11-11T17:19:46.108228Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518755353679429:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.108270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518755353679429:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:46.108511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518755353679412:2095] [node 6] ICP35 event processing took too much time PendingConnection:278(107360186ns) {HandleHandshakeStatus:329(107357452ns) {DropIncomingHandshake:441(13723ns) DropOutgoingHandshake:456(523ns) SwitchToState:178(327ns) ProcessPendingSessionEvents:495(32934128ns) {ForwardSessionEventToSession:612(7872873ns) ForwardSessionEventToSession:612(5046ns) ForwardSessionEventToSession:612(10341523ns) ForwardSessionEventToSession:612(4723078ns) ForwardSessionEventToSession:612(6627925ns) ForwardSessionEventToSession:612(8461ns) ForwardSessionEventToSession:612(3940ns) ForwardSessionEventToSession:612(3320528ns) ForwardSessionEventToSession:612(4599ns) ForwardSessionEventToSession:612(2433ns)}}}
2025-11-11T17:19:46.109359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518755353679443:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:46.111966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518751058712057:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.112025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:46.116733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518751058712057:2048] [node 1] ICP22 created new session: [3:7571518759648646749:2048]
2025-11-11T17:19:46.116784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518759648646749:2048] [node 1] ICS09 handshake done sender: [3:7571518755353679443:2120] self: [3:7571518759648646744:0] peer: [1:7571518755734543065:0] socket: 384 qp: -1
2025-11-11T17:19:46.116821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518759648646749:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:46.116879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:45.955159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:46.109796Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518760715732018:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.894205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518757043505474:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:45.886408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518755734543072:2107] [node 4] ICS10 traffic start
2025-11-11T17:19:46.116599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518756420764666:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.961911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518755717699992:2068] [node 4] ICP22 created new session: [6:7571518755717700039:2068]
2025-11-11T17:19:45.963153Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518755717700023:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:45.963347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518755717699965:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:45.963381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:45.963415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518755717699965:2064] [node 5] ICP22 created new session: [6:7571518755717700040:2064]
2025-11-11T17:19:45.963452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518755717700040:2064] [node 5] ICS09 handshake done sender: [6:7571518755717700015:2111] self: [6:7571518755717700028:0] peer: [5:7571518756420764679:0] socket: 345 qp: -1
2025-11-11T17:19:45.963478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518755717700040:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:45.963500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:45.969927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518755717700039:2068] [node 4] ICS09 handshake done sender: [6:7571518755717700017:2113] self: [6:7571518755717700026:0] peer: [4:7571518757043505463:0] socket: 347 qp: -1
2025-11-11T17:19:45.978277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518755717700039:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:45.995686Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518755717700016:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:45.996880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:46.004146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518751422732653:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.010319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518751422732651:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.010377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:46.010424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518751422732651:2048] [node 1] ICP22 created new session: [6:7571518760012667341:2048]
2025-11-11T17:19:46.010461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518760012667341:2048] [node 1] ICS09 handshake done sender: [6:7571518755717700016:2112] self: [6:7571518755717700035:0] peer: [1:7571518755734543061:0] socket: 346 qp: -1
2025-11-11T17:19:46.010493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518760012667341:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:46.010528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:46.017867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:46.017908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518751422732653:2050] [node 3] ICP22 created new session: [6:7571518760012667340:2050]
2025-11-11T17:19:46.022031Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:46.029876Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518755717700030:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.037751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518760012667340:2050] [node 3] ICS09 handshake done sender: [6:7571518755717700023:2116] self: [6:7571518755717700029:0] peer: [3:7571518755353679426:0] socket: 356 qp: -1
2025-11-11T17:19:46.038945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518755717699992:2068] [node 4] ICP35 event processing took too much time PendingConnection:278(92962588ns) {HandleHandshakeStatus:329(92960113ns) {DropIncomingHandshake:441(1255ns) DropOutgoingHandshake:456(5614ns) SwitchToState:178(1771ns) ProcessPendingSessionEvents:495(8870598ns) {ForwardSessionEventToSession:612(8817575ns) ForwardSessionEventToSession:612(6819ns) ForwardSessionEventToSession:612(6333ns) ForwardSessionEventToSession:612(3024ns) ForwardSessionEventToSession:612(2467ns) ForwardSessionEventToSession:612(8181ns) ForwardSessionEventToSession:612(2532ns) ForwardSessionEventToSession:612(2068ns) ForwardSessionEventToSession:612(9378ns) ForwardSessionEventToSession:612(2129ns)}}}
2025-11-11T17:19:46.045912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518760012667340:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:46.045978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:46.053709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518751422732652:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:46.064872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518751422732652:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16049590ns) {IncomingHandshake:261(16047081ns) {DropIncomingHandshake:441(843ns) PrepareNewSessionHandshake:476(1481ns) {SwitchToState:178(607ns)} GenerateSessionVirtualId:988(724ns) IssueIncomingHandshakeReply:193(65724ns) {SwitchToState:178(517ns)}}}
2025-11-11T17:19:46.115684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518760012667345:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33049
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:46.154060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518755353679429:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:45.894242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:46.157765Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518755717700030:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:46.118194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518760715732024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.090084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518752710266325:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(93239390ns) {HandleHandshakeStatus:329(93237235ns) {DropIncomingHandshake:441(2011ns) DropOutgoingHandshake:456(10911ns) SwitchToState:178(132ns) ProcessPendingSessionEvents:495(110679ns) {ForwardSessionEventToSession:612(64195ns) ForwardSessionEventToSession:612(3194ns) ForwardSessionEventToSession:612(8277ns) ForwardSessionEventToSession:612(3580ns) ForwardSessionEventToSession:612(5141ns) ForwardSessionEventToSession:612(3762ns) ForwardSessionEventToSession:612(3300ns) ForwardSessionEventToSession:612(5438ns) ForwardSessionEventToSession:612(2618ns) ForwardSessionEventToSession:612(3048ns)}}}
2025-11-11T17:19:45.886458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:45.898396Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518757043505468:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.887449Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518755734543057:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:46.093935Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233702:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.923926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543062:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.923954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518755734543062:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:45.924585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518751439575692:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:45.924636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:45.924690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518751439575692:2048] [node 2] ICP22 created new session: [1:7571518755734543074:2048]
2025-11-11T17:19:45.924732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518755734543074:2048] [node 2] ICS09 handshake done sender: [1:7571518755734543057:2112] self: [1:7571518755734543056:0] peer: [2:7571518757005233705:0] socket: 336 qp: -1
2025-11-11T17:19:45.924769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518755734543074:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:45.924815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:45.939091Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518755734543062:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:45.962201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518755734543053:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:45.962241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:45.962277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518755734543053:2109] [node 6] ICP22 created new session: [1:7571518755734543079:2109]
2025-11-11T17:19:45.962319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518755734543079:2109] [node 6] ICS09 handshake done sender: [1:7571518755734543062:2113] self: [1:7571518755734543061:0] peer: [6:7571518755717700035:0] socket: 339 qp: -1
2025-11-11T17:19:45.962351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518755734543079:2109] [node 6] ICS10 traffic start
2025-11-11T17:19:45.962384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:45.985656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543071:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.986111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518751439575692:2048] [node 2] ICP18 handshake (actor [1:7571518755734543071:2116]) is too old
2025-11-11T17:19:45.998103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518755734543060:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(328062351ns) {Configure:104(328054302ns) {ProcessConfigured:135(23359816ns) {StartInitialHandshake:157(23356984ns) {DropHandshakes:469(1818ns) {DropIncomingHandshake:441(509ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:476(4023ns) {SwitchToState:178(1591ns)}}}}}
2025-11-11T17:19:45.999281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518755734543083:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-11T17:19:46.014019Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:46.064910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543066:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.064946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518755734543066:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:46.077926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518755734543081:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.077979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518755734543081:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:46.095537Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518755734543066:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:46.113702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518751439575693:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.113771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:46.114036Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518755734543081:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:46.120233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518751439575693:2049] [node 3] ICP22 created new session: [1:7571518760029510382:2049]
2025-11-11T17:19:46.126812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518755734543060:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.138432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:46.129680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518760029510382:2049] [node 3] ICS09 handshake done sender: [1:7571518755734543066:2115] self: [1:7571518755734543065:0] peer: [3:7571518759648646744:0] socket: 377 qp: -1
2025-11-11T17:19:46.140109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518755734543060:2108] [node 5] ICP22 created new session: [1:7571518760029510383:2108]
2025-11-11T17:19:46.143217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518760029510382:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:46.143287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:46.153880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518760029510383:2108] [node 5] ICS09 handshake done sender: [1:7571518755734543081:2117] self: [1:7571518755734543078:0] peer: [5:7571518760715732019:0] socket: 352 qp: -1
2025-11-11T17:19:46.154930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518760029510383:2108] [node 5] ICS10 traffic start
2025-11-11T17:19:46.160841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:45.898443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518757043505446:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.133776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518760715732017:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:46.169878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518755353679420:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.093975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518757005233702:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:46.170029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518751422732652:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.134060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518756420764719:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:46.134244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518752125797341:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.134340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:46.134412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518752125797341:2049] [node 2] ICP22 created new session: [5:7571518760715732025:2049]
2025-11-11T17:19:46.134469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518760715732025:2049] [node 2] ICS09 handshake done sender: [5:7571518760715732017:2118] self: [5:7571518760715732021:0] peer: [2:7571518757005233713:0] socket: 387 qp: -1
2025-11-11T17:19:46.134507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518760715732025:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:46.134563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:46.135045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518752125797340:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.135115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:46.135154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518752125797340:2048] [node 1] ICP22 created new session: [5:7571518760715732027:2048]
2025-11-11T17:19:46.135188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518760715732027:2048] [node 1] ICS09 handshake done sender: [5:7571518756420764719:2116] self: [5:7571518760715732019:0] peer: [1:7571518755734543078:0] socket: 386 qp: -1
2025-11-11T17:19:46.135222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518760715732027:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:46.135254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:46.138088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518760715732029:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.153936Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518760715732018:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:46.160726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518760715732030:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50248
2025-11-11T17:19:46.170278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518756420764666:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.170341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:46.170385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518756420764666:2078] [node 4] ICP22 created new session: [5:7571518760715732031:2078]
2025-11-11T17:19:46.170424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518760715732031:2078] [node 4] ICS09 handshake done sender: [5:7571518760715732018:2119] self: [5:7571518760715732023:0] peer: [4:7571518757043505484:0] socket: 389 qp: -1
2025-11-11T17:19:46.170460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518760715732031:2078] [node 4] ICS10 traffic start
2025-11-11T17:19:46.170505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:46.191307Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:46.193705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518752125797342:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518760715732016:2117] held: no
2025-11-11T17:19:46.200809Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518760715732016:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:46.202436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518756420764666:2078] [node 4] ICP35 event processing took too much time PendingConnection:278(31923104ns) {HandleHandshakeStatus:329(31918130ns) {DropIncomingHandshake:441(241ns) DropOutgoingHandshake:456(11793ns) SwitchToState:178(112ns) ProcessPendingSessionEvents:495(14787569ns) {ForwardSessionEventToSession:612(14734783ns) ForwardSessionEventToSession:612(11529ns) ForwardSessionEventToSession:612(2532ns) ForwardSessionEventToSession:612(1896ns) ForwardSessionEventToSession:612(5344ns) ForwardSessionEventToSession:612(2681ns) ForwardSessionEventToSession:612(2700ns) ForwardSessionEventToSession:612(7507ns) ForwardSessionEventToSession:612(2966ns) ForwardSessionEventToSession:612(5535ns)}}}
2025-11-11T17:19:46.170102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:46.170160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518751422732652:2049] [node 2] ICP22 created new session: [6:7571518760012667346:2049]
2025-11-11T17:19:46.171504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518751422732653:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(118854497ns) {HandleHandshakeStatus:329(118852076ns) {DropIncomingHandshake:441(1599ns) DropOutgoingHandshake:456(5806ns) SwitchToState:178(1074ns) ProcessPendingSessionEvents:495(29493112ns) {ForwardSessionEventToSession:612(14721734ns) ForwardSessionEventToSession:612(7265ns) ForwardSessionEventToSession:612(7792813ns) ForwardSessionEventToSession:612(21182ns) ForwardSessionEventToSession:612(3430ns) ForwardSessionEventToSession:612(1984146ns) ForwardSessionEventToSession:612(4383ns) ForwardSessionEventToSession:612(3297ns) ForwardSessionEventToSession:612(4928416ns) ForwardSessionEventToSession:612(3568ns)}}}
2025-11-11T17:19:46.171583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518751422732653:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:46.181926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518760012667346:2049] [node 2] ICS09 handshake done sender: [6:7571518755717700030:2118] self: [6:7571518760012667343:0] peer: [2:7571518757005233701:0] socket: 363 qp: -1
2025-11-11T17:19:46.189779Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17223) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:46.190063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518760012667346:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:46.190118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:46.190584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518751422732652:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20326944ns) {HandleHandshakeStatus:329(20324921ns) {DropIncomingHandshake:441(728ns) DropOutgoingHandshake:456(4677ns) SwitchToState:178(300ns) ProcessPendingSessionEvents:495(105443ns) {ForwardSessionEventToSession:612(52669ns) ForwardSessionEventToSession:612(11015ns) ForwardSessionEventToSession:612(2886ns) ForwardSessionEventToSession:612(2520ns) ForwardSessionEventToSession:612(8578ns) ForwardSessionEventToSession:612(2792ns) ForwardSessionEventToSession:612(2608ns) ForwardSessionEventToSession:612(8074ns) ForwardSessionEventToSession:612(2429ns) ForwardSessionEventToSession:612(5115ns)}}}
2025-11-11T17:19:46.190689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518751422732652:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:46.194253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:46.201101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518755353679420:2097] [node 5] ICP22 created new session: [3:7571518759648646752:2097]
2025-11-11T17:19:46.208431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518759648646752:2097] [node 5] ICS09 handshake done sender: [3:7571518755353679429:2114] self: [3:7571518755353679428:0] peer: [5:7571518760715732020:0] socket: 372 qp: -1
2025-11-11T17:19:46.224735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518759648646752:2097] [node 5] ICS10 traffic start
2025-11-11T17:19:46.224793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:46.225223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518755353679420:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(55149048ns) {HandleHandshakeStatus:329(55145719ns) {DropIncomingHandshake:441(28709ns) DropOutgoingHandshake:456(404ns) SwitchToState:178(435ns) ProcessPendingSessionEvents:495(85410ns) {ForwardSessionEventToSession:612(44473ns) ForwardSessionEventToSession:612(3506ns) ForwardSessionEventToSession:612(4868ns) ForwardSessionEventToSession:612(7378ns) ForwardSessionEventToSession:612(4494ns) ForwardSessionEventToSession:612(1862ns) ForwardSessionEventToSession:612(2800ns) ForwardSessionEventToSession:612(3870ns) ForwardSessionEventToSession:612(2777ns) ForwardSessionEventToSession:612(2544ns)}}}
2025-11-11T17:19:45.898487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:45.898529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518757043505446:2089] [node 6] ICP22 created new session: [4:7571518757043505477:2089]
2025-11-11T17:19:45.898571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518757043505477:2089] [node 6] ICS09 handshake done sender: [4:7571518757043505464:2112] self: [4:7571518757043505463:0] peer: [6:7571518755717700026:0] socket: 344 qp: -1
2025-11-11T17:19:45.898602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518757043505477:2089] [node 6] ICS10 traffic start
2025-11-11T17:19:45.898633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:45.923093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518752748538099:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(448919307ns) {Configure:104(437003013ns) {ProcessConfigured:135(25143482ns) {StartInitialHandshake:157(25140182ns) {DropHandshakes:469(3127ns) {DropIncomingHandshake:441(897ns) DropOutgoingHandshake:456(330ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:476(3738ns) {SwitchToState:178(1685ns)}}}}}
2025-11-11T17:19:45.935525Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518757043505465:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:45.935694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518752748538100:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:45.943999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518757043505482:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:45.944096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518757043505483:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:45.949567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518757043505460:2083] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(285566182ns) {Configure:104(285559887ns) {ProcessConfigured:135(88056ns) {StartInitialHandshake:157(83419ns) {DropHandshakes:469(2854ns) {DropIncomingHandshake:441(634ns) DropOutgoingHandshake:456(566ns)} GenerateSessionVirtualId:988(800ns) PrepareNewSessionHandshake:476(1512ns) {SwitchToState:178(539ns)}}}}}
2025-11-11T17:19:45.975053Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518757043505465:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:45.977932Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:45.978385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518757043505446:2089] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:46.005402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518752748538100:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.005474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:46.241869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518751439575693:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(69639308ns) {HandleHandshakeStatus:329(69635365ns) {DropIncomingHandshake:441(22499ns) DropOutgoingHandshake:456(471ns) SwitchToState:178(165ns) ProcessPendingSessionEvents:495(39870668ns) {ForwardSessionEventToSession:612(12860430ns) ForwardSessionEventToSession:612(4502520ns) ForwardSessionEventToSession:612(3300ns) ForwardSessionEventToSession:612(4858020ns) ForwardSessionEventToSession:612(8297ns) ForwardSessionEventToSession:612(2733ns) ForwardSessionEventToSession:612(518315ns) ForwardSessionEventToSession:612(5420082ns) ForwardSessionEventToSession:612(7196ns) ForwardSessionEventToSession:612(11656756ns)}}}
2025-11-11T17:19:46.005532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518752748538100:2050] [node 3] ICP22 created new session: [4:7571518757043505487:2050]
2025-11-11T17:19:46.005577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518757043505487:2050] [node 3] ICS09 handshake done sender: [4:7571518757043505465:2113] self: [4:7571518757043505481:0] peer: [3:7571518755353679421:0] socket: 343 qp: -1
2025-11-11T17:19:46.015035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518757043505487:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:46.019027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518761338472784:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:46.032622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:46.063552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518752748538100:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(57935177ns) {HandleHandshakeStatus:329(57932634ns) {DropIncomingHandshake:441(354ns) DropOutgoingHandshake:456(5833ns) SwitchToState:178(265ns) ProcessPendingSessionEvents:495(8193208ns) {ForwardSessionEventToSession:612(8132168ns) ForwardSessionEventToSession:612(11434ns) ForwardSessionEventToSession:612(3540ns) ForwardSessionEventToSession:612(7106ns) ForwardSessionEventToSession:612(2994ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(10719ns) ForwardSessionEventToSession:612(3488ns) ForwardSessionEventToSession:612(3388ns) ForwardSessionEventToSession:612(5443ns)}}}
2025-11-11T17:19:46.099253Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518761338472784:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.106027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518752748538099:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:46.118355Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518757043505485:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.118379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518757043505485:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:46.119481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518761338472788:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13660
2025-11-11T17:19:46.125266Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518757043505485:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:46.157686Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518761338472784:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:46.160466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518757043505460:2083] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.169397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:46.169606Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17223) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:46.177484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518752748538099:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.177538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:46.177584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518752748538099:2049] [node 2] ICP22 created new session: [4:7571518761338472791:2049]
2025-11-11T17:19:46.177647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518761338472791:2049] [node 2] ICS09 handshake done sender: [4:7571518761338472784:2121] self: [4:7571518761338472787:0] peer: [2:7571518757005233707:0] socket: 388 qp: -1
2025-11-11T17:19:46.177684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518761338472791:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:46.177720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:46.177913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518752748538099:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:46.182305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518757043505460:2083] [node 5] ICP22 created new session: [4:7571518761338472790:2083]
2025-11-11T17:19:46.200935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518761338472790:2083] [node 5] ICS09 handshake done sender: [4:7571518757043505485:2120] self: [4:7571518757043505484:0] peer: [5:7571518760715732023:0] socket: 348 qp: -1
2025-11-11T17:19:46.208769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518761338472790:2083] [node 5] ICS10 traffic start
2025-11-11T17:19:46.221681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:46.105829Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233714:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.105866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518757005233714:2126] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:46.114485Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518757005233708:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:46.114509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518757005233708:2123] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:46.116438Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518757005233714:2126] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:46.123032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518757005233702:2121] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:46.126060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518757005233708:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:46.149681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518757005233697:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.150538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518757005233698:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.150623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:46.150693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518757005233698:2098] [node 6] ICP22 created new session: [2:7571518761300201024:2098]
2025-11-11T17:19:46.150748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518761300201024:2098] [node 6] ICS09 handshake done sender: [2:7571518757005233702:2121] self: [2:7571518757005233701:0] peer: [6:7571518760012667343:0] socket: 361 qp: -1
2025-11-11T17:19:46.150780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518761300201024:2098] [node 6] ICS10 traffic start
2025-11-11T17:19:46.150812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:46.151207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518757005233683:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:46.151285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:46.161662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:46.164791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518757005233697:2097] [node 5] ICP22 created new session: [2:7571518761300201027:2097]
2025-11-11T17:19:46.169443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518757005233683:2096] [node 4] ICP22 created new session: [2:7571518761300201026:2096]
2025-11-11T17:19:46.169671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518761300201027:2097] [node 5] ICS09 handshake done sender: [2:7571518757005233714:2126] self: [2:7571518757005233713:0] peer: [5:7571518760715732021:0] socket: 380 qp: -1
2025-11-11T17:19:46.182365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518761300201026:2096] [node 4] ICS09 handshake done sender: [2:7571518757005233708:2123] self: [2:7571518757005233707:0] peer: [4:7571518761338472787:0] socket: 350 qp: -1
2025-11-11T17:19:46.182414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518761300201026:2096] [node 4] ICS10 traffic start
2025-11-11T17:19:46.182474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:46.184925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518761300201027:2097] [node 5] ICS10 traffic start
2025-11-11T17:19:46.184982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:46.185404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518757005233697:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(35567230ns) {HandleHandshakeStatus:329(35558191ns) {DropIncomingHandshake:441(16336ns) DropOutgoingHandshake:456(447ns) SwitchToState:178(566ns) ProcessPendingSessionEvents:495(115253ns) {ForwardSessionEventToSession:612(66729ns) ForwardSessionEventToSession:612(3620ns) ForwardSessionEventToSession:612(3645ns) ForwardSessionEventToSession:612(6673ns) ForwardSessionEventToSession:612(2480ns) ForwardSessionEventToSession:612(7979ns) ForwardSessionEventToSession:612(2662ns) ForwardSessionEventToSession:612(3154ns) ForwardSessionEventToSession:612(7362ns) ForwardSessionEventToSession:612(3205ns)}}}
2025-11-11T17:19:46.227034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518757005233683:2096] [node 4] ICP35 event processing took too much time PendingConnection:278(75596398ns) {HandleHandshakeStatus:329(75593845ns) {DropIncomingHandshake:441(8088ns) DropOutgoingHandshake:456(302ns) SwitchToState:178(100ns) ProcessPendingSessionEvents:495(9047725ns) {ForwardSessionEventToSession:612(8995445ns) ForwardSessionEventToSession:612(5050ns) ForwardSessionEventToSession:612(3036ns) ForwardSessionEventToSession:612(4946ns) ForwardSessionEventToSession:612(2305ns) ForwardSessionEventToSession:612(3501ns) ForwardSessionEventToSession:612(9430ns) ForwardSessionEventToSession:612(2209ns) ForwardSessionEventToSession:612(7380ns) ForwardSessionEventToSession:612(2763ns)}}}
2025-11-11T17:19:46.258610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518755734543060:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(131552033ns) {HandleHandshakeStatus:329(131549211ns) {DropIncomingHandshake:441(447ns) DropOutgoingHandshake:456(908ns) SwitchToState:178(609ns) ProcessPendingSessionEvents:495(64133230ns) {ForwardSessionEventToSession:612(26640518ns) ForwardSessionEventToSession:612(5939ns) ForwardSessionEventToSession:612(3675896ns) ForwardSessionEventToSession:612(3459544ns) ForwardSessionEventToSession:612(3097ns) ForwardSessionEventToSession:612(1987ns) ForwardSessionEventToSession:612(24168510ns) ForwardSessionEventToSession:612(3667ns) ForwardSessionEventToSession:612(6140012ns) ForwardSessionEventToSession:612(4773ns)}}}
2025-11-11T17:19:46.227877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518752125797342:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:46.245999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:46.250750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518752125797342:2050] [node 3] ICP22 created new session: [5:7571518760715732033:2050]
2025-11-11T17:19:46.250815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518760715732033:2050] [node 3] ICS09 handshake done sender: [5:7571518760715732016:2117] self: [5:7571518760715732020:0] peer: [3:7571518755353679428:0] socket: 375 qp: -1
2025-11-11T17:19:46.250864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518760715732033:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:46.250936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:46.251141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518752125797342:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(30039147ns) {HandleHandshakeStatus:329(30036997ns) {DropIncomingHandshake:441(2062ns) DropOutgoingHandshake:456(790ns) SwitchToState:178(4128ns) ProcessPendingSessionEvents:495(323ns)}}
2025-11-11T17:19:46.257989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518757043505460:2083] [node 5] ICP35 event processing took too much time PendingConnection:278(109204970ns) {HandleHandshakeStatus:329(109201316ns) {DropIncomingHandshake:441(28603ns) DropOutgoingHandshake:456(375ns) SwitchToState:178(494ns) ProcessPendingSessionEvents:495(5644227ns) {ForwardSessionEventToSession:612(5582048ns) ForwardSessionEventToSession:612(15526ns) ForwardSessionEventToSession:612(3620ns) ForwardSessionEventToSession:612(2905ns) ForwardSessionEventToSession:612(7805ns) ForwardSessionEventToSession:612(4555ns) ForwardSessionEventToSession:612(2244ns) ForwardSessionEventToSession:612(8524ns) ForwardSessionEventToSession:612(2666ns) ForwardSessionEventToSession:612(5245ns)}}}
2025-11-11T17:19:46.303494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:46.303758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518761300201016:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-11-11T17:19:46.303797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518752710266325:2048] [node 1] ICP30 unregister session Session# [2:7571518761300201016:2048] VirtualId# [2:7571518757005233705:0]
2025-11-11T17:19:46.303831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518761300201016:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.303918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:46.303968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:46.305365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:46.305620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518759648646749:2048] [node 1] ICS01 socket: 384 reason# EndOfStream
2025-11-11T17:19:46.305658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518751058712057:2048] [node 1] ICP30 unregister session Session# [3:7571518759648646749:2048] VirtualId# [3:7571518759648646744:0]
2025-11-11T17:19:46.305683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518759648646749:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.305743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:46.305779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:46.320561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:46.321705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:46.321918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518760715732027:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-11T17:19:46.321956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518752125797340:2048] [node 1] ICP30 unregister session Session# [5:7571518760715732027:2048] VirtualId# [5:7571518760715732019:0]
2025-11-11T17:19:46.321983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518760715732027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.322066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:46.322106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:46.320799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518757043505474:2048] [node 1] ICS01 socket: 338 reason# EndOfStream
2025-11-11T17:19:46.320848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518752748538098:2048] [node 1] ICP30 unregister session Session# [4:7571518757043505474:2048] VirtualId# [4:7571518757043505467:0]
2025-11-11T17:19:46.320882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518757043505474:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.320966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:46.321012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:46.322986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:46.323198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518760012667341:2048] [node 1] ICS01 socket: 346 reason# EndOfStream
2025-11-11T17:19:46.323243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518751422732651:2048] [node 1] ICP30 unregister session Session# [6:7571518760012667341:2048] VirtualId# [6:7571518755717700035:0]
2025-11-11T17:19:46.323272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518760012667341:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.323347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:46.323393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:46.430471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:46.430734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518759648646746:2049] [node 2] ICS07 socket disconnect 334 reason# EndOfStream
2025-11-11T17:19:46.430764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518759648646746:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.430866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:46.430912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518759648646746:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:46.437807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:46.437803Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17223) Couldn't connect to any resolved address
2025-11-11T17:19:46.438765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518751058712058:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17223) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:46.438954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518759648646746:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:46.438982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518751058712058:2049] [node 2] ICP30 unregister session Session# [3:7571518759648646746:2049] VirtualId# [3:7571518755353679438:0]
2025-11-11T17:19:46.439021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:46.439048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518751058712058:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17223) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:46.439061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17223) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:46.438253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518761338472791:2049] [node 2] ICS01 socket: 388 reason# EndOfStream
2025-11-11T17:19:46.438298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518752748538099:2049] [node 2] ICP30 unregister session Session# [4:7571518761338472791:2049] VirtualId# [4:7571518761338472787:0]
2025-11-11T17:19:46.438334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518761338472791:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.438402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:46.438442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:46.438015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:46.438564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518760715732025:2049] [node 2] ICS01 socket: 387 reason# EndOfStream
2025-11-11T17:19:46.438591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518752125797341:2049] [node 2] ICP30 unregister session Session# [5:7571518760715732025:2049] VirtualId# [5:7571518760715732021:0]
2025-11-11T17:19:46.438614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518760715732025:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.438655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:46.438688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:46.441324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:46.441539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518760012667346:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-11-11T17:19:46.441575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518751422732652:2049] [node 2] ICP30 unregister session Session# [6:7571518760012667346:2049] VirtualId# [6:7571518760012667343:0]
2025-11-11T17:19:46.441621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518760012667346:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.441690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:46.441740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:46.449920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518751058712058:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:46.609277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:46.609539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518757043505487:2050] [node 3] ICS01 socket: 343 reason# EndOfStream
2025-11-11T17:19:46.609611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518752748538100:2050] [node 3] ICP30 unregister session Session# [4:7571518757043505487:2050] VirtualId# [4:7571518757043505481:0]
2025-11-11T17:19:46.609651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518757043505487:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.609735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:46.609800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:46.609292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:46.609537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518760012667340:2050] [node 3] ICS07 socket disconnect 356 reason# EndOfStream
2025-11-11T17:19:46.609567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518760012667340:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.609676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:46.609714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518760012667340:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:46.614558Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:46.614825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518760715732033:2050] [node 3] ICS01 socket: 375 reason# EndOfStream
2025-11-11T17:19:46.614866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518752125797342:2050] [node 3] ICP30 unregister session Session# [5:7571518760715732033:2050] VirtualId# [5:7571518760715732020:0]
2025-11-11T17:19:46.614899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518760715732033:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.614972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:46.615017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:46.615908Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11798) Couldn't connect to any resolved address
2025-11-11T17:19:46.616065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518751422732653:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11798) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:46.616115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518760012667340:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:46.616146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518751422732653:2050] [node 3] ICP30 unregister session Session# [6:7571518760012667340:2050] VirtualId# [6:7571518755717700029:0]
2025-11-11T17:19:46.616183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:46.616236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518751422732653:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:46.616251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:46.623313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518751422732653:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:46.701795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:46.702020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518760715732031:2078] [node 4] ICS01 socket: 389 reason# EndOfStream
2025-11-11T17:19:46.702061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518756420764666:2078] [node 4] ICP30 unregister session Session# [5:7571518760715732031:2078] VirtualId# [5:7571518760715732023:0]
2025-11-11T17:19:46.702097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518760715732031:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.702194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:46.702238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:46.702331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:46.702504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518755717700039:2068] [node 4] ICS01 socket: 347 reason# EndOfStream
2025-11-11T17:19:46.702535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518755717699992:2068] [node 4] ICP30 unregister session Session# [6:7571518755717700039:2068] VirtualId# [6:7571518755717700026:0]
2025-11-11T17:19:46.702561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518755717700039:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.702612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:46.702648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:46.933745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:46.933996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518755717700040:2064] [node 5] ICS01 socket: 345 reason# EndOfStream
2025-11-11T17:19:46.934038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518755717699965:2064] [node 5] ICP30 unregister session Session# [6:7571518755717700040:2064] VirtualId# [6:7571518755717700028:0]
2025-11-11T17:19:46.934074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518755717700040:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:46.934168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:46.934213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:47.163416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518763413607163:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:47.163476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518763413607164:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:47.217942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518763538240469:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:47.217987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518763538240470:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:47.238214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518764740342490:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:47.238260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518764740342491:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:47.301833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518764215942739:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:47.301890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518764215942740:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:47.301919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518764215942741:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:47.339958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518764865207373:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:47.340009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518764865207374:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:47.340037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518764865207375:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:47.629697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518764865207386:2059] [node 4] ICP01 ready to work
2025-11-11T17:19:47.631782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518763538240470:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(69784772ns) {Configure:104(57910106ns) {ProcessConfigured:135(84100ns) {StartInitialHandshake:157(78508ns) {DropHandshakes:469(2376ns) {DropIncomingHandshake:441(552ns) DropOutgoingHandshake:456(485ns)} GenerateSessionVirtualId:988(753ns) PrepareNewSessionHandshake:476(1739ns) {SwitchToState:178(596ns)}}}}}
2025-11-11T17:19:47.637315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518764740342531:2071] [node 4] ICP01 ready to work
2025-11-11T17:19:47.632246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518763538240503:2066] [node 6] ICP01 ready to work
2025-11-11T17:19:47.369104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518765917073719:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:47.673729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518763413607216:2072] [node 6] ICP01 ready to work
2025-11-11T17:19:47.749948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518764740342555:2072] [node 5] ICP01 ready to work
2025-11-11T17:19:47.715610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518764215942739:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(129569742ns) {Configure:104(129564881ns) {ProcessConfigured:135(76668ns) {StartInitialHandshake:157(73283ns) {DropHandshakes:469(2597ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(541ns)} GenerateSessionVirtualId:988(976ns) PrepareNewSessionHandshake:476(1597ns) {SwitchToState:178(596ns)}}}}}
2025-11-11T17:19:47.369144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518765917073720:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:47.767201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518763413607226:2074] [node 4] ICP01 ready to work
2025-11-11T17:19:47.632515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518763538240504:2067] [node 5] ICP01 ready to work
2025-11-11T17:19:47.632736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518763538240505:2068] [node 4] ICP01 ready to work
2025-11-11T17:19:47.750660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518763538240469:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(117549351ns) {Configure:104(117544902ns) {ProcessConfigured:135(63006ns) {StartInitialHandshake:157(59802ns) {DropHandshakes:469(2806ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(523ns)} GenerateSessionVirtualId:988(546ns) PrepareNewSessionHandshake:476(1777ns) {SwitchToState:178(617ns)}}}}}
2025-11-11T17:19:47.762167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518764740342556:2073] [node 6] ICP01 ready to work
2025-11-11T17:19:47.369171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518765917073721:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:47.838878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518763538240505:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(85362254ns) {Configure:104(85358339ns) {ProcessConfigured:135(57600ns) {StartInitialHandshake:157(55764ns) {DropHandshakes:469(2170ns) {DropIncomingHandshake:441(464ns) DropOutgoingHandshake:456(480ns)} GenerateSessionVirtualId:988(1080ns) PrepareNewSessionHandshake:476(1269ns) {SwitchToState:178(565ns)}}}}}
2025-11-11T17:19:47.861805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518763413607228:2087] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:47.553707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518765917073732:2059] [node 5] ICP01 ready to work
2025-11-11T17:19:47.855108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518764740342490:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(89614273ns) {Configure:104(89610754ns) {ProcessConfigured:135(63285ns) {StartInitialHandshake:157(61262ns) {DropHandshakes:469(2641ns) {DropIncomingHandshake:441(580ns) DropOutgoingHandshake:456(432ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:476(1712ns) {SwitchToState:178(758ns)}}}}}
2025-11-11T17:19:47.870839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518763538240548:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62584
2025-11-11T17:19:47.828280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518764215942740:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(112394361ns) {Configure:104(112389031ns) {ProcessConfigured:135(26319464ns) {StartInitialHandshake:157(26316665ns) {DropHandshakes:469(2812ns) {DropIncomingHandshake:441(537ns) DropOutgoingHandshake:456(495ns)} GenerateSessionVirtualId:988(576ns) PrepareNewSessionHandshake:476(5638ns) {SwitchToState:178(1959ns)}}}}}
2025-11-11T17:19:47.830262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518764215942808:2065] [node 5] ICP01 ready to work
2025-11-11T17:19:47.830546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518764215942809:2070] [node 6] ICP01 ready to work
2025-11-11T17:19:47.870906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518763538240549:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62584
2025-11-11T17:19:47.950291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518765917073732:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(389989863ns) {Configure:104(389977694ns) {ProcessConfigured:135(65916ns) {StartInitialHandshake:157(63447ns) {DropHandshakes:469(2944ns) {DropIncomingHandshake:441(692ns) DropOutgoingHandshake:456(574ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:476(1830ns) {SwitchToState:178(557ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:47.941997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518764865207375:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(228036479ns) {Configure:104(228029692ns) {ProcessConfigured:135(11150653ns) {StartInitialHandshake:157(11147323ns) {DropHandshakes:469(2784ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(679ns)} GenerateSessionVirtualId:988(668ns) PrepareNewSessionHandshake:476(5577ns) {SwitchToState:178(1764ns)}}}}}
2025-11-11T17:19:47.944099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518764865207441:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:47.950822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518765917073785:2070] [node 4] ICP01 ready to work
2025-11-11T17:19:47.961944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518763413607163:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(84920165ns) {Configure:104(84916823ns) {ProcessConfigured:135(15991638ns) {StartInitialHandshake:157(15989040ns) {DropHandshakes:469(1784ns) {DropIncomingHandshake:441(326ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(366ns) PrepareNewSessionHandshake:476(10247ns) {SwitchToState:178(3518ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:47.994476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518764865207441:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37583138ns) {Configure:104(37579328ns) {ProcessConfigured:135(7132311ns) {StartInitialHandshake:157(7127132ns) {DropHandshakes:469(2794ns) {DropIncomingHandshake:441(441ns) DropOutgoingHandshake:456(504ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:476(3556ns) {SwitchToState:178(1649ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:47.979088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518765917073720:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26219200ns) {Configure:104(26215982ns) {ProcessConfigured:135(79411ns) {StartInitialHandshake:157(77016ns) {DropHandshakes:469(2630ns) {DropIncomingHandshake:441(864ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(710ns) PrepareNewSessionHandshake:476(1749ns) {SwitchToState:178(616ns)}}}}}
2025-11-11T17:19:48.009838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518765917073785:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25098253ns) {Configure:104(25094764ns) {ProcessConfigured:135(15899352ns) {StartInitialHandshake:157(15897254ns) {DropHandshakes:469(1578ns) {DropIncomingHandshake:441(318ns) DropOutgoingHandshake:456(235ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:476(4237ns) {SwitchToState:178(1892ns)}}}}}
2025-11-11T17:19:48.010407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518765917073786:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.010914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041091:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.027305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174746:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:47.994181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518763538240550:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62584
2025-11-11T17:19:48.037647Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518763538240502:2079] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.037691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518763538240502:2079] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:47.998220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518764740342570:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40964
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:47.986145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518763413607239:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5418
2025-11-11T17:19:48.006243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518764215942817:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.076330Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518763538240549:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.082061Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518765917073786:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.085508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518765917073720:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.077514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518767833207847:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62584
2025-11-11T17:19:48.077565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518767833207848:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62584
2025-11-11T17:19:48.086304Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518763538240502:2079] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:47.998453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518764740342571:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40964
2025-11-11T17:19:48.020404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518768510910114:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
2025-11-11T17:19:48.020528Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518764215942817:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.020636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518764215942741:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.026527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518768510910116:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
2025-11-11T17:19:48.027850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518768510910117:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
2025-11-11T17:19:48.034595Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518764215942817:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:48.034777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518764215942741:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.034833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:48.034893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518764215942741:2050] [node 3] ICP22 created new session: [4:7571518768510910118:2050]
2025-11-11T17:19:48.034945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518768510910118:2050] [node 3] ICS09 handshake done sender: [4:7571518764215942817:2115] self: [4:7571518768510910115:0] peer: [3:7571518764740342557:0] socket: 348 qp: -1
2025-11-11T17:19:48.034989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518768510910118:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:48.035030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:47.986358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518763413607240:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5418
2025-11-11T17:19:48.051514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174748:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.008251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518764740342570:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:47.986415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518763413607241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5418
2025-11-11T17:19:48.141637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518770212041091:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.141808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518765917073719:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.008382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518764740342491:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.166291Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518763538240548:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.166495Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518763538240538:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.166514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518763538240538:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.167069Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518763538240550:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.167109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518763538240470:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.167166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:48.167202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518763538240470:2049] [node 3] ICP22 created new session: [2:7571518767833207851:2049]
2025-11-11T17:19:48.167245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518767833207851:2049] [node 3] ICS09 handshake done sender: [2:7571518763538240502:2079] self: [2:7571518763538240501:0] peer: [3:7571518769035309868:0] socket: 337 qp: -1
2025-11-11T17:19:48.167286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518767833207851:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:48.167321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:48.008951Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607239:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.179114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518767833207847:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.157725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518768510910114:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.014063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518769035309869:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40964
2025-11-11T17:19:48.119031Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607240:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.178437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174749:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.020974Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518764740342558:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.180610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518764215942739:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.119223Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607241:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.165652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607230:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.165697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518763413607230:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.179912Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518763413607230:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:48.201946Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607234:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.201977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518763413607234:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.202780Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607232:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.202797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518763413607232:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.203326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518763413607216:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.203396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:48.203456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518763413607216:2072] [node 6] ICP22 created new session: [1:7571518767708574545:2072]
2025-11-11T17:19:48.203508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518767708574545:2072] [node 6] ICS09 handshake done sender: [1:7571518763413607230:2110] self: [1:7571518763413607229:0] peer: [6:7571518770212041096:0] socket: 354 qp: -1
2025-11-11T17:19:48.203545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518767708574545:2072] [node 6] ICS10 traffic start
2025-11-11T17:19:48.203578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.206713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.207174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041098:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.207485Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518770212041091:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:48.207703Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518765917073786:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:48.207854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518765917073719:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.207904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:48.207956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518765917073719:2048] [node 1] ICP22 created new session: [6:7571518770212041099:2048]
2025-11-11T17:19:48.208001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518770212041099:2048] [node 1] ICS09 handshake done sender: [6:7571518770212041091:2116] self: [6:7571518770212041096:0] peer: [1:7571518763413607229:0] socket: 359 qp: -1
2025-11-11T17:19:48.208033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518770212041099:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:48.208062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:48.208498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518765917073720:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.208534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:48.208587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518765917073720:2049] [node 2] ICP22 created new session: [6:7571518770212041101:2049]
2025-11-11T17:19:48.208621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518770212041101:2049] [node 2] ICS09 handshake done sender: [6:7571518765917073786:2111] self: [6:7571518770212041095:0] peer: [2:7571518763538240537:0] socket: 350 qp: -1
2025-11-11T17:19:48.208646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518770212041101:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:48.208670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:48.210245Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518763413607234:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:48.179299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518763538240469:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.194087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518769160174748:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.193881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518763538240538:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:48.180929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518768510910117:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.020996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518764740342558:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.033991Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518764740342558:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:48.078611Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518764740342571:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.078879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518764740342531:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.078920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:48.078965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518764740342531:2071] [node 4] ICP22 created new session: [3:7571518769035309873:2071]
2025-11-11T17:19:48.079016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518769035309873:2071] [node 4] ICS09 handshake done sender: [3:7571518764740342558:2111] self: [3:7571518764740342557:0] peer: [4:7571518768510910115:0] socket: 341 qp: -1
2025-11-11T17:19:48.079046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518769035309873:2071] [node 4] ICS10 traffic start
2025-11-11T17:19:48.079076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:48.080020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518764740342531:2071] [node 4] ICP18 handshake (actor [3:7571518764740342571:2117]) is too old
2025-11-11T17:19:48.080136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518769035309871:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40964
2025-11-11T17:19:48.080270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518769035309872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40964
2025-11-11T17:19:48.146003Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518764740342570:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:48.154234Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518769035309869:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.154278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518764740342491:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.220214Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518769160174746:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.220495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174750:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.220571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518764865207386:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.154352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:48.154396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518764740342491:2049] [node 2] ICP22 created new session: [3:7571518769035309876:2049]
2025-11-11T17:19:48.154438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518769035309876:2049] [node 2] ICS09 handshake done sender: [3:7571518764740342570:2116] self: [3:7571518769035309868:0] peer: [2:7571518763538240501:0] socket: 347 qp: -1
2025-11-11T17:19:48.154470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518769035309876:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:48.154522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:48.210141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518769035309871:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.218777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518769035309878:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40964
2025-11-11T17:19:48.221203Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518763413607232:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:48.206412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518768510910123:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
2025-11-11T17:19:48.246339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518763413607163:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.246468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:48.246521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518763413607163:2048] [node 2] ICP22 created new session: [1:7571518767708574550:2048]
2025-11-11T17:19:48.246559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518767708574550:2048] [node 2] ICS09 handshake done sender: [1:7571518763413607234:2112] self: [1:7571518763413607233:0] peer: [2:7571518767833207856:0] socket: 356 qp: -1
2025-11-11T17:19:48.246594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518767708574550:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:48.246625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:48.247083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518763413607226:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.247150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:48.247190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518763413607226:2074] [node 4] ICP22 created new session: [1:7571518767708574552:2074]
2025-11-11T17:19:48.247254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518767708574552:2074] [node 4] ICS09 handshake done sender: [1:7571518763413607232:2111] self: [1:7571518763413607231:0] peer: [4:7571518768510910121:0] socket: 355 qp: -1
2025-11-11T17:19:48.247311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518767708574552:2074] [node 4] ICS10 traffic start
2025-11-11T17:19:48.247364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:48.225808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518764215942811:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.225090Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518767833207848:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.225830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518764215942811:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.238872Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518768510910114:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:48.239084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518764215942739:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.239126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:48.239168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518764215942739:2048] [node 1] ICP22 created new session: [4:7571518768510910125:2048]
2025-11-11T17:19:48.239224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518768510910125:2048] [node 1] ICS09 handshake done sender: [4:7571518768510910114:2116] self: [4:7571518768510910121:0] peer: [1:7571518763413607231:0] socket: 358 qp: -1
2025-11-11T17:19:48.239255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518768510910125:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:48.239314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:48.240720Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518764215942811:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:48.265480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518764215942808:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.265579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:48.265683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518764215942808:2065] [node 5] ICP22 created new session: [4:7571518768510910127:2065]
2025-11-11T17:19:48.265730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518768510910127:2065] [node 5] ICS09 handshake done sender: [4:7571518764215942811:2113] self: [4:7571518764215942810:0] peer: [5:7571518769160174751:0] socket: 346 qp: -1
2025-11-11T17:19:48.265769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518768510910127:2065] [node 5] ICS10 traffic start
2025-11-11T17:19:48.265814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:48.225335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518763538240503:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.225388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:48.225460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518763538240503:2066] [node 6] ICP22 created new session: [2:7571518767833207859:2066]
2025-11-11T17:19:48.225504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518767833207859:2066] [node 6] ICS09 handshake done sender: [2:7571518763538240538:2110] self: [2:7571518763538240537:0] peer: [6:7571518770212041095:0] socket: 339 qp: -1
2025-11-11T17:19:48.225537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518767833207859:2066] [node 6] ICS10 traffic start
2025-11-11T17:19:48.225580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:48.258153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518763538240503:2066] [node 6] ICP35 event processing took too much time PendingConnection:278(32565229ns) {HandleHandshakeStatus:329(32562478ns) {DropIncomingHandshake:441(11943ns) DropOutgoingHandshake:456(459ns) SwitchToState:178(226ns) ProcessPendingSessionEvents:495(32154767ns) {ForwardSessionEventToSession:612(48030ns) ForwardSessionEventToSession:612(2813ns) ForwardSessionEventToSession:612(165793ns) ForwardSessionEventToSession:612(3319ns) ForwardSessionEventToSession:612(2926ns) ForwardSessionEventToSession:612(16145209ns) ForwardSessionEventToSession:612(3854ns) ForwardSessionEventToSession:612(8005276ns) ForwardSessionEventToSession:612(7756531ns) ForwardSessionEventToSession:612(3767ns)}}}
2025-11-11T17:19:48.279420Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607236:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.279458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518763413607236:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.280866Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518763413607236:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:48.277856Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518769035309878:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.278044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518764740342490:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.280640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518769035309883:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40964
2025-11-11T17:19:48.269583Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:48.298123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518769160174749:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.298257Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:48.299179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518763413607164:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.299260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:48.299317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518763413607164:2049] [node 3] ICP22 created new session: [1:7571518767708574556:2049]
2025-11-11T17:19:48.299364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518767708574556:2049] [node 3] ICS09 handshake done sender: [1:7571518763413607236:2113] self: [1:7571518763413607235:0] peer: [3:7571518769035309882:0] socket: 384 qp: -1
2025-11-11T17:19:48.299394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518767708574556:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:48.299467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.293878Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518769035309878:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.312670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174753:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.303329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518767833207858:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62584
2025-11-11T17:19:48.310371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518763538240469:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518763538240550:2117] held: no
2025-11-11T17:19:48.310668Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518763538240550:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:48.311784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518763538240469:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.311824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:48.311869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518763538240469:2048] [node 1] ICP22 created new session: [2:7571518767833207863:2048]
2025-11-11T17:19:48.311916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518767833207863:2048] [node 1] ICS09 handshake done sender: [2:7571518763538240550:2117] self: [2:7571518767833207856:0] peer: [1:7571518763413607233:0] socket: 357 qp: -1
2025-11-11T17:19:48.311946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518767833207863:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:48.311975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:48.305991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518768510910130:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
2025-11-11T17:19:48.301397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041103:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.309862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041104:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.313533Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518770212041104:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.313658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518765917073785:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.326260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518764865207386:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518769160174748:2116] held: no
2025-11-11T17:19:48.326666Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518769160174748:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:48.333707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174754:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.333916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518764865207375:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.333973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518764865207386:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.334060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:48.334101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518764865207386:2059] [node 4] ICP22 created new session: [5:7571518769160174756:2059]
2025-11-11T17:19:48.334139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518769160174756:2059] [node 4] ICS09 handshake done sender: [5:7571518769160174748:2116] self: [5:7571518769160174751:0] peer: [4:7571518764215942810:0] socket: 371 qp: -1
2025-11-11T17:19:48.334173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518769160174756:2059] [node 4] ICS10 traffic start
2025-11-11T17:19:48.334213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:48.327292Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518764215942813:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.327328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518764215942813:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.348410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518767708574558:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5418
2025-11-11T17:19:48.327487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518764740342490:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.327549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:48.327586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518764740342490:2048] [node 1] ICP22 created new session: [3:7571518769035309884:2048]
2025-11-11T17:19:48.327648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518769035309884:2048] [node 1] ICS09 handshake done sender: [3:7571518769035309878:2121] self: [3:7571518769035309882:0] peer: [1:7571518763413607235:0] socket: 386 qp: -1
2025-11-11T17:19:48.327685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518769035309884:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:48.327747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:48.341851Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:48.347463Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518768510910130:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.347627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518768510910131:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.349164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.346111Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518769160174754:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.346240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518769160174750:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.346476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518764865207373:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.346540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518764865207374:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.350021Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518770212041103:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.350140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518765917073721:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.349863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518763538240540:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.349906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518763538240540:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.351046Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518763538240540:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:48.350591Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518763413607238:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.350622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518763413607238:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.351834Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518764215942813:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.365208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174761:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.357745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518763413607238:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:48.362129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518763538240504:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.362216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:48.366546Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518764740342564:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.366580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518764740342564:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.367058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518764740342490:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:48.367222Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518764740342566:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.367239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518764740342566:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.365754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518763413607228:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.365829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:48.365872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518763413607228:2087] [node 5] ICP22 created new session: [1:7571518767708574560:2087]
2025-11-11T17:19:48.365920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518767708574560:2087] [node 5] ICS09 handshake done sender: [1:7571518763413607238:2114] self: [1:7571518763413607237:0] peer: [5:7571518769160174759:0] socket: 351 qp: -1
2025-11-11T17:19:48.365953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518767708574560:2087] [node 5] ICS10 traffic start
2025-11-11T17:19:48.365982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:48.362267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518763538240504:2067] [node 5] ICP22 created new session: [2:7571518767833207866:2067]
2025-11-11T17:19:48.362317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518767833207866:2067] [node 5] ICS09 handshake done sender: [2:7571518763538240540:2111] self: [2:7571518763538240539:0] peer: [5:7571518769160174760:0] socket: 379 qp: -1
2025-11-11T17:19:48.362354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518767833207866:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:48.362396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.373707Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518770212041104:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:48.386080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518765917073785:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.386130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:48.386181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518765917073785:2070] [node 4] ICP22 created new session: [6:7571518770212041110:2070]
2025-11-11T17:19:48.386231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518770212041110:2070] [node 4] ICS09 handshake done sender: [6:7571518770212041104:2120] self: [6:7571518770212041106:0] peer: [4:7571518764215942812:0] socket: 390 qp: -1
2025-11-11T17:19:48.386266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518770212041110:2070] [node 4] ICS10 traffic start
2025-11-11T17:19:48.386298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:48.388101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041109:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.375854Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518764740342564:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:48.381494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174762:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.399105Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518768510910131:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.399203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518764215942809:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.399266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:48.399321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518764215942809:2070] [node 6] ICP22 created new session: [4:7571518768510910134:2070]
2025-11-11T17:19:48.399370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518768510910134:2070] [node 6] ICS09 handshake done sender: [4:7571518764215942813:2114] self: [4:7571518764215942812:0] peer: [6:7571518770212041106:0] socket: 362 qp: -1
2025-11-11T17:19:48.399405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518768510910134:2070] [node 6] ICS10 traffic start
2025-11-11T17:19:48.399446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:48.418020Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518764740342566:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:48.418932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518764215942740:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.429821Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518770212041103:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:48.430162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518765917073721:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.430228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:48.430275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518765917073721:2050] [node 3] ICP22 created new session: [6:7571518770212041112:2050]
2025-11-11T17:19:48.430318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518770212041112:2050] [node 3] ICS09 handshake done sender: [6:7571518770212041103:2119] self: [6:7571518770212041108:0] peer: [3:7571518764740342565:0] socket: 338 qp: -1
2025-11-11T17:19:48.430356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518770212041112:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:48.430409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:48.405369Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30964) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.419186Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518767708574558:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.442297Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518769160174750:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:48.419757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518763413607228:2087] [node 5] ICP18 handshake (actor [1:7571518767708574558:2118]) is too old
2025-11-11T17:19:48.425960Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518763538240543:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.445774Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518764865207374:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30964) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518769160174750:2118] held: no
2025-11-11T17:19:48.445889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518764865207374:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.445941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:48.445982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518764865207374:2049] [node 2] ICP22 created new session: [5:7571518769160174764:2049]
2025-11-11T17:19:48.446024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518769160174764:2049] [node 2] ICS09 handshake done sender: [5:7571518769160174750:2118] self: [5:7571518769160174760:0] peer: [2:7571518763538240539:0] socket: 380 qp: -1
2025-11-11T17:19:48.446056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518769160174764:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:48.446094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:48.446449Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:48.442864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518764740342555:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.442937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:48.442978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518764740342555:2072] [node 5] ICP22 created new session: [3:7571518769035309888:2072]
2025-11-11T17:19:48.443021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518769035309888:2072] [node 5] ICS09 handshake done sender: [3:7571518764740342564:2114] self: [3:7571518764740342563:0] peer: [5:7571518769160174755:0] socket: 349 qp: -1
2025-11-11T17:19:48.443050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518769035309888:2072] [node 5] ICS10 traffic start
2025-11-11T17:19:48.443089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:48.445235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518764740342556:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.445275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:48.445311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518764740342556:2073] [node 6] ICP22 created new session: [3:7571518769035309890:2073]
2025-11-11T17:19:48.445348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518769035309890:2073] [node 6] ICS09 handshake done sender: [3:7571518764740342566:2115] self: [3:7571518764740342565:0] peer: [6:7571518770212041108:0] socket: 387 qp: -1
2025-11-11T17:19:48.445375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518769035309890:2073] [node 6] ICS10 traffic start
2025-11-11T17:19:48.445401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:48.437968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518768510910137:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11580
2025-11-11T17:19:48.442722Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518768510910131:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:48.443485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518764215942740:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.443542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:48.443587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518764215942740:2049] [node 2] ICP22 created new session: [4:7571518768510910138:2049]
2025-11-11T17:19:48.443635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518768510910138:2049] [node 2] ICS09 handshake done sender: [4:7571518768510910131:2121] self: [4:7571518768510910136:0] peer: [2:7571518763538240542:0] socket: 392 qp: -1
2025-11-11T17:19:48.443664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518768510910138:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:48.443692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:48.426006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518763538240543:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:48.438267Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518763538240543:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:48.447868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518763538240505:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.447911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:48.447948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518763538240505:2068] [node 4] ICP22 created new session: [2:7571518767833207869:2068]
2025-11-11T17:19:48.447982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518767833207869:2068] [node 4] ICS09 handshake done sender: [2:7571518763538240543:2114] self: [2:7571518763538240542:0] peer: [4:7571518768510910136:0] socket: 353 qp: -1
2025-11-11T17:19:48.448021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518767833207869:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:48.448044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:48.452138Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518769160174754:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:48.452582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518764865207373:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518769160174754:2120] held: no
2025-11-11T17:19:48.452674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518764865207373:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.452708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:48.452750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518764865207373:2048] [node 1] ICP22 created new session: [5:7571518769160174766:2048]
2025-11-11T17:19:48.452792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518769160174766:2048] [node 1] ICS09 handshake done sender: [5:7571518769160174754:2120] self: [5:7571518769160174759:0] peer: [1:7571518763413607237:0] socket: 352 qp: -1
2025-11-11T17:19:48.452825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518769160174766:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:48.452853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:48.453189Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1184) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.454816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518769160174763:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9826
2025-11-11T17:19:48.455393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518764865207375:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1184) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518769160174749:2117] held: no
2025-11-11T17:19:48.455663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518769160174749:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:48.461994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518764865207375:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.462049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:48.462098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518764865207375:2050] [node 3] ICP22 created new session: [5:7571518769160174769:2050]
2025-11-11T17:19:48.462141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518769160174769:2050] [node 3] ICS09 handshake done sender: [5:7571518769160174749:2117] self: [5:7571518769160174755:0] peer: [3:7571518764740342563:0] socket: 377 qp: -1
2025-11-11T17:19:48.462173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518769160174769:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:48.462204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.467384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041114:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.469857Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518770212041114:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.470065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518765917073732:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:48.473794Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518764865207447:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:48.473829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518764865207447:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:48.478027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518770212041116:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39729
2025-11-11T17:19:48.481551Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518764865207447:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:48.482202Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518770212041114:2123] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:48.482487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518764865207441:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:48.482550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:48.482592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518764865207441:2068] [node 6] ICP22 created new session: [5:7571518769160174772:2068]
2025-11-11T17:19:48.482633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518769160174772:2068] [node 6] ICS09 handshake done sender: [5:7571518764865207447:2114] self: [5:7571518764865207446:0] peer: [6:7571518770212041115:0] socket: 340 qp: -1
2025-11-11T17:19:48.482672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518769160174772:2068] [node 6] ICS10 traffic start
2025-11-11T17:19:48.482704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:48.483562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518765917073732:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:48.483635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:48.483671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518765917073732:2059] [node 5] ICP22 created new session: [6:7571518770212041117:2059]
2025-11-11T17:19:48.483711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518770212041117:2059] [node 5] ICS09 handshake done sender: [6:7571518770212041114:2123] self: [6:7571518770212041115:0] peer: [5:7571518764865207446:0] socket: 344 qp: -1
2025-11-11T17:19:48.483741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518770212041117:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:48.483764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:48.505640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:48.505864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518767833207863:2048] [node 1] ICS01 socket: 357 reason# EndOfStream
2025-11-11T17:19:48.505903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518763538240469:2048] [node 1] ICP30 unregister session Session# [2:7571518767833207863:2048] VirtualId# [2:7571518767833207856:0]
2025-11-11T17:19:48.505934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518767833207863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.505984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:48.506018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:48.507446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:48.507661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518769035309884:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-11T17:19:48.507694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518764740342490:2048] [node 1] ICP30 unregister session Session# [3:7571518769035309884:2048] VirtualId# [3:7571518769035309882:0]
2025-11-11T17:19:48.507718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518769035309884:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.507775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:48.507801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:48.524706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:48.524953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518770212041099:2048] [node 1] ICS01 socket: 359 reason# EndOfStream
2025-11-11T17:19:48.524990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518765917073719:2048] [node 1] ICP30 unregister session Session# [6:7571518770212041099:2048] VirtualId# [6:7571518770212041096:0]
2025-11-11T17:19:48.525026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518770212041099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.525088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:48.525126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:48.525205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:48.525387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518768510910125:2048] [node 1] ICS01 socket: 358 reason# EndOfStream
2025-11-11T17:19:48.525417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518764215942739:2048] [node 1] ICP30 unregister session Session# [4:7571518768510910125:2048] VirtualId# [4:7571518768510910121:0]
2025-11-11T17:19:48.525436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518768510910125:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.525470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:48.525495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:48.532100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:48.532331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518769160174766:2048] [node 1] ICS01 socket: 352 reason# EndOfStream
2025-11-11T17:19:48.532365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518764865207373:2048] [node 1] ICP30 unregister session Session# [5:7571518769160174766:2048] VirtualId# [5:7571518769160174759:0]
2025-11-11T17:19:48.532400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518769160174766:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.532483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:48.532519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:48.623313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:48.623563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518769035309876:2049] [node 2] ICS01 socket: 347 reason# EndOfStream
2025-11-11T17:19:48.623601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518764740342491:2049] [node 2] ICP30 unregister session Session# [3:7571518769035309876:2049] VirtualId# [3:7571518769035309868:0]
2025-11-11T17:19:48.623642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518769035309876:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.623712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:48.623756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:48.641816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:48.642069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518770212041101:2049] [node 2] ICS01 socket: 350 reason# EndOfStream
2025-11-11T17:19:48.642112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518765917073720:2049] [node 2] ICP30 unregister session Session# [6:7571518770212041101:2049] VirtualId# [6:7571518770212041095:0]
2025-11-11T17:19:48.642143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518770212041101:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.642223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:48.642266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:48.641923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:48.642176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518769160174764:2049] [node 2] ICS01 socket: 380 reason# EndOfStream
2025-11-11T17:19:48.642213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518764865207374:2049] [node 2] ICP30 unregister session Session# [5:7571518769160174764:2049] VirtualId# [5:7571518769160174760:0]
2025-11-11T17:19:48.642247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518769160174764:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.642355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:48.642397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:48.645876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:48.646270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518768510910138:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-11-11T17:19:48.646340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518764215942740:2049] [node 2] ICP30 unregister session Session# [4:7571518768510910138:2049] VirtualId# [4:7571518768510910136:0]
2025-11-11T17:19:48.646404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518768510910138:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.646575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:48.646645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:48.741809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:48.742054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518769160174769:2050] [node 3] ICS01 socket: 377 reason# EndOfStream
2025-11-11T17:19:48.742093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518764865207375:2050] [node 3] ICP30 unregister session Session# [5:7571518769160174769:2050] VirtualId# [5:7571518769160174755:0]
2025-11-11T17:19:48.742123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518769160174769:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.742200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:48.742239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:48.742327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:48.742510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518770212041112:2050] [node 3] ICS01 socket: 338 reason# EndOfStream
2025-11-11T17:19:48.742538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518765917073721:2050] [node 3] ICP30 unregister session Session# [6:7571518770212041112:2050] VirtualId# [6:7571518770212041108:0]
2025-11-11T17:19:48.742568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518770212041112:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.742611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:48.742646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:48.745929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:48.793816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:48.794053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518769160174756:2059] [node 4] ICS01 socket: 371 reason# EndOfStream
2025-11-11T17:19:48.794087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518764865207386:2059] [node 4] ICP30 unregister session Session# [5:7571518769160174756:2059] VirtualId# [5:7571518769160174751:0]
2025-11-11T17:19:48.794119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518769160174756:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.794202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:48.794240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:48.794319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:48.794502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518770212041110:2070] [node 4] ICS01 socket: 390 reason# EndOfStream
2025-11-11T17:19:48.794529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518765917073785:2070] [node 4] ICP30 unregister session Session# [6:7571518770212041110:2070] VirtualId# [6:7571518770212041106:0]
2025-11-11T17:19:48.794555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518770212041110:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.794592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:48.794624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:48.917895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:48.918227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518770212041117:2059] [node 5] ICS01 socket: 344 reason# EndOfStream
2025-11-11T17:19:48.918291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518765917073732:2059] [node 5] ICP30 unregister session Session# [6:7571518770212041117:2059] VirtualId# [6:7571518770212041115:0]
2025-11-11T17:19:48.918346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518770212041117:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:48.918498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:48.918569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:49.293786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518774832077930:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:49.293842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518774832077931:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:49.329894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518771595033427:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:49.329940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518771595033428:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:49.385855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518775024956706:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:49.385896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518775024956707:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:49.434125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518772307615950:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:49.434166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518772307615951:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:49.434192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518772307615952:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:49.506115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518774130899399:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:49.506216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518774130899400:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:49.509449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518774130899401:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:49.557908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518774047509040:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:49.557948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518774047509041:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:49.557971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518774047509042:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:49.781251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518772307615971:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:49.781521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518772307615972:2067] [node 6] ICP01 ready to work
2025-11-11T17:19:49.874023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518774832077991:2094] [node 6] ICP01 ready to work
2025-11-11T17:19:49.874369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518774832077992:2098] [node 4] ICP01 ready to work
2025-11-11T17:19:49.887919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518771595033474:2072] [node 6] ICP01 ready to work
2025-11-11T17:19:49.894260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518772307615971:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(104354040ns) {Configure:104(104348574ns) {ProcessConfigured:135(6319540ns) {StartInitialHandshake:157(6316130ns) {DropHandshakes:469(2341ns) {DropIncomingHandshake:441(371ns) DropOutgoingHandshake:456(577ns)} GenerateSessionVirtualId:988(659ns) PrepareNewSessionHandshake:476(4903ns) {SwitchToState:178(2059ns)}}}}}
2025-11-11T17:19:49.877774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518774832077997:2096] [node 5] ICP01 ready to work
2025-11-11T17:19:49.877975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518775024956726:2062] [node 4] ICP01 ready to work
2025-11-11T17:19:49.930294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518771595033476:2076] [node 5] ICP01 ready to work
2025-11-11T17:19:49.941750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518774130899399:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(140028094ns) {Configure:104(140021077ns) {ProcessConfigured:135(5581520ns) {StartInitialHandshake:157(5578702ns) {DropHandshakes:469(2987ns) {DropIncomingHandshake:441(603ns) DropOutgoingHandshake:456(318ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:476(4292ns) {SwitchToState:178(2132ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:49.953956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518775024956726:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45954986ns) {Configure:104(45946658ns) {ProcessConfigured:135(11932451ns) {StartInitialHandshake:157(11929096ns) {DropHandshakes:469(2189ns) {DropIncomingHandshake:441(643ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(606ns) PrepareNewSessionHandshake:476(3790ns) {SwitchToState:178(955ns)}}}}}
2025-11-11T17:19:49.948684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518771595033478:2080] [node 4] ICP01 ready to work
2025-11-11T17:19:49.974781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518775024956727:2064] [node 6] ICP01 ready to work
2025-11-11T17:19:49.964462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518771595033474:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23808524ns) {Configure:104(23803686ns) {ProcessConfigured:135(74755ns) {StartInitialHandshake:157(66332ns) {DropHandshakes:469(2790ns) {DropIncomingHandshake:441(813ns) DropOutgoingHandshake:456(493ns)} GenerateSessionVirtualId:988(748ns) PrepareNewSessionHandshake:476(1344ns) {SwitchToState:178(594ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.009077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518771595033428:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(44404892ns) {Configure:104(44400284ns) {ProcessConfigured:135(14201634ns) {StartInitialHandshake:157(14198644ns) {DropHandshakes:469(2982ns) {DropIncomingHandshake:441(1074ns) DropOutgoingHandshake:456(428ns)} GenerateSessionVirtualId:988(1353ns) PrepareNewSessionHandshake:476(5539ns) {SwitchToState:178(2504ns)}}}}}
2025-11-11T17:19:50.098518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518771595033478:2080] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(89085317ns) {Configure:104(89077355ns) {ProcessConfigured:135(70649ns) {StartInitialHandshake:157(68288ns) {DropHandshakes:469(2460ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(378ns)} GenerateSessionVirtualId:988(1022ns) PrepareNewSessionHandshake:476(1743ns) {SwitchToState:178(489ns)}}}}}
2025-11-11T17:19:50.133927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518774832077992:2098] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(255883956ns) {Configure:104(255879820ns) {ProcessConfigured:135(11586665ns) {StartInitialHandshake:157(11583825ns) {DropHandshakes:469(2368ns) {DropIncomingHandshake:441(628ns) DropOutgoingHandshake:456(497ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(5031ns) {SwitchToState:178(2256ns)}}}}}
2025-11-11T17:19:50.137802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518771595033476:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(151995330ns) {Configure:104(151985975ns) {ProcessConfigured:135(6578179ns) {StartInitialHandshake:157(6575173ns) {DropHandshakes:469(2014ns) {DropIncomingHandshake:441(396ns) DropOutgoingHandshake:456(343ns)} GenerateSessionVirtualId:988(351ns) PrepareNewSessionHandshake:476(3489ns) {SwitchToState:178(660ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:49.982604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518772307615982:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-11T17:19:50.066446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518772307615982:2071] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.178492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518772307615952:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(357838174ns) {Configure:104(357833254ns) {ProcessConfigured:135(41769192ns) {StartInitialHandshake:157(25808633ns) {DropHandshakes:469(3084ns) {DropIncomingHandshake:441(483ns) DropOutgoingHandshake:456(443ns)} GenerateSessionVirtualId:988(2672ns) PrepareNewSessionHandshake:476(2822ns) {SwitchToState:178(1422ns)}}}}}
2025-11-11T17:19:50.230869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518772307615952:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.143967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518779127045300:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27428
2025-11-11T17:19:50.101989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518779319924030:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53590
2025-11-11T17:19:50.177968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518775890000787:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3646
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.244971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518776602583294:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-11T17:19:50.197776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518775890000787:2100] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.193987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866710:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
2025-11-11T17:19:50.144160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518779127045301:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27428
2025-11-11T17:19:50.161927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518775024956727:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(171945465ns) {Configure:104(171939609ns) {ProcessConfigured:135(15811316ns) {StartInitialHandshake:157(15807795ns) {DropHandshakes:469(2180ns) {DropIncomingHandshake:441(424ns) DropOutgoingHandshake:456(496ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:476(4610ns) {SwitchToState:178(1884ns)}}}}}
2025-11-11T17:19:50.334060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518778342476412:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.198621Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518775890000779:2097] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.162225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518779319924030:2069] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.259771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518776602583297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-11T17:19:50.278115Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518776602583297:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.293872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518772307615952:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(32298265ns) {IncomingHandshake:261(32296604ns) {DropIncomingHandshake:441(562ns) PrepareNewSessionHandshake:476(2694ns) {SwitchToState:178(1357ns)} GenerateSessionVirtualId:988(429ns) IssueIncomingHandshakeReply:193(58216ns) {SwitchToState:178(509ns)}}}
2025-11-11T17:19:50.382166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518776602583294:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.144253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518779127045302:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27428
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.406744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518774047509040:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(96520560ns) {Configure:104(96511078ns) {ProcessConfigured:135(50579ns) {StartInitialHandshake:157(45050ns) {DropHandshakes:469(2017ns) {DropIncomingHandshake:441(343ns) DropOutgoingHandshake:456(503ns)} GenerateSessionVirtualId:988(623ns) PrepareNewSessionHandshake:476(1431ns) {SwitchToState:178(507ns)}}}}}
2025-11-11T17:19:50.408160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476365:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.408337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476393:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.408445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476394:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.408993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476417:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.162600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518779319924033:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53590
2025-11-11T17:19:50.164111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518775024956707:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.182687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518779319924033:2071] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.182803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518775024956706:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.184313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518779319924037:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53590
2025-11-11T17:19:50.185013Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518779319924033:2071] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:50.185184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518775024956706:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.185233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:50.185276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518775024956706:2048] [node 1] ICP22 created new session: [3:7571518779319924038:2048]
2025-11-11T17:19:50.185315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518779319924038:2048] [node 1] ICS09 handshake done sender: [3:7571518779319924033:2071] self: [3:7571518779319924036:0] peer: [1:7571518774832077993:0] socket: 362 qp: -1
2025-11-11T17:19:50.185344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518779319924038:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:50.185375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:50.291580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518779319924077:2075] [node 5] ICP01 ready to work
2025-11-11T17:19:50.409854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518775024956729:2067] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.409889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518775024956729:2067] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.401829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518772307615951:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.144344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518779127045303:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27428
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.198656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518775890000779:2097] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.242234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518771595033427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.246516Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518775890000779:2097] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:50.246798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518771595033428:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.246850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:50.246892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518771595033428:2049] [node 3] ICP22 created new session: [2:7571518775890000798:2049]
2025-11-11T17:19:50.246935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518775890000798:2049] [node 3] ICS09 handshake done sender: [2:7571518775890000779:2097] self: [2:7571518771595033482:0] peer: [3:7571518779319924035:0] socket: 352 qp: -1
2025-11-11T17:19:50.246964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518775890000798:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:50.247000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:50.366298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518775890000809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3646
2025-11-11T17:19:50.374052Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518775890000787:2100] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:50.397773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518771595033427:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.397848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:50.397903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518771595033427:2048] [node 1] ICP22 created new session: [2:7571518775890000810:2048]
2025-11-11T17:19:50.397955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518775890000810:2048] [node 1] ICS09 handshake done sender: [2:7571518775890000787:2100] self: [2:7571518775890000797:0] peer: [1:7571518774832077995:0] socket: 364 qp: -1
2025-11-11T17:19:50.397996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518775890000810:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:50.398028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:50.442310Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518775890000781:2098] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.442335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518775890000781:2098] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.443781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518775890000812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3646
2025-11-11T17:19:50.449798Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518775890000781:2098] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:50.425705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518778342476426:2109] [node 5] ICP01 ready to work
2025-11-11T17:19:50.450863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518778342476365:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.450947Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518778342476393:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.451018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518778342476394:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.166790Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518779127045302:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.181665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518779127045301:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.183237Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518774832077994:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.183257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518774832077994:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.184476Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518774832077994:2108] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:50.186163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518774832077991:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(261962653ns) {Configure:104(261960161ns) {ProcessConfigured:135(29806762ns) {StartInitialHandshake:157(29805259ns) {DropHandshakes:469(2247ns) {DropIncomingHandshake:441(415ns) DropOutgoingHandshake:456(394ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:476(2661ns) {SwitchToState:178(117ns)}}}}}
2025-11-11T17:19:50.198778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518774832077931:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.198843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:50.229940Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518779127045303:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.230044Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518779127045300:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.262921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518774832077931:2049] [node 3] ICP22 created new session: [1:7571518779127045310:2049]
2025-11-11T17:19:50.317792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518779127045310:2049] [node 3] ICS09 handshake done sender: [1:7571518774832077994:2108] self: [1:7571518774832077993:0] peer: [3:7571518779319924036:0] socket: 359 qp: -1
2025-11-11T17:19:50.322044Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518774832077996:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.322070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518774832077996:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.349670Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518774832077996:2109] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:50.373711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518779127045310:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:50.373774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:50.414210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518774832077931:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(227771682ns) {HandleHandshakeStatus:329(215859721ns) {DropIncomingHandshake:441(15196ns) DropOutgoingHandshake:456(423ns) SwitchToState:178(186ns) ProcessPendingSessionEvents:495(101916ns) {ForwardSessionEventToSession:612(58976ns) ForwardSessionEventToSession:612(5611ns) ForwardSessionEventToSession:612(2818ns) ForwardSessionEventToSession:612(5356ns) ForwardSessionEventToSession:612(5482ns) ForwardSessionEventToSession:612(2455ns) ForwardSessionEventToSession:612(6394ns) ForwardSessionEventToSession:612(3317ns) ForwardSessionEventToSession:612(2786ns)}}}
2025-11-11T17:19:50.414290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518774832077930:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.414342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:50.414388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518774832077930:2048] [node 2] ICP22 created new session: [1:7571518779127045318:2048]
2025-11-11T17:19:50.414426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518779127045318:2048] [node 2] ICS09 handshake done sender: [1:7571518774832077996:2109] self: [1:7571518774832077995:0] peer: [2:7571518775890000797:0] socket: 360 qp: -1
2025-11-11T17:19:50.414463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518779127045318:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:50.414493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:50.450485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518779127045320:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27428
2025-11-11T17:19:50.453638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518772307615950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.453829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518772307615950:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(47923448ns) {IncomingHandshake:261(23916597ns) {DropIncomingHandshake:441(416ns) PrepareNewSessionHandshake:476(1657ns) {SwitchToState:178(636ns)} GenerateSessionVirtualId:988(737ns) IssueIncomingHandshakeReply:193(15678583ns) {SwitchToState:178(2625ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.463212Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518779127045296:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.463251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518779127045296:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.453798Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518778342476417:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.458830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518779319924084:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53590
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.485935Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:50.465927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518774130899400:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(259985570ns) {Configure:104(259980771ns) {ProcessConfigured:135(4052906ns) {StartInitialHandshake:157(4049526ns) {DropHandshakes:469(3008ns) {DropIncomingHandshake:441(815ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:476(3691ns) {SwitchToState:178(1490ns)}}}}}
2025-11-11T17:19:50.466411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518778425866764:2069] [node 6] ICP01 ready to work
2025-11-11T17:19:50.466582Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518778425866710:2060] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.466868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518778425866765:2111] [node 4] ICP01 ready to work
2025-11-11T17:19:50.470779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518774130899401:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(264822830ns) {Configure:104(264818984ns) {ProcessConfigured:135(36940ns) {StartInitialHandshake:157(35512ns) {DropHandshakes:469(1493ns) {DropIncomingHandshake:441(461ns) DropOutgoingHandshake:456(267ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:476(755ns) {SwitchToState:178(258ns)}}}}}
2025-11-11T17:19:50.471088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866766:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
2025-11-11T17:19:50.471225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866767:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
2025-11-11T17:19:50.476409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518778425866765:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.474069Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518779127045296:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.497668Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518778425866766:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.497890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518778425866767:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.498027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518774130899399:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.497972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518778342476426:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22775148ns) {Configure:104(22769819ns) {ProcessConfigured:135(12080899ns) {StartInitialHandshake:157(12078886ns) {DropHandshakes:469(1720ns) {DropIncomingHandshake:441(385ns) DropOutgoingHandshake:456(362ns)} GenerateSessionVirtualId:988(553ns) PrepareNewSessionHandshake:476(3595ns) {SwitchToState:178(1815ns)}}}}}
2025-11-11T17:19:50.498505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518778342476412:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.498585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518774047509041:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.498633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518774047509042:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.498679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518774047509040:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.499067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476429:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.471777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518771595033478:2080] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.506012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476434:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.506589Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518778342476417:2125] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:50.506775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518774047509040:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.506829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:50.506867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518774047509040:2048] [node 1] ICP22 created new session: [6:7571518778342476435:2048]
2025-11-11T17:19:50.506907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518778342476435:2048] [node 1] ICS09 handshake done sender: [6:7571518778342476417:2125] self: [6:7571518778342476433:0] peer: [1:7571518779127045294:0] socket: 375 qp: -1
2025-11-11T17:19:50.506954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518778342476435:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:50.506990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:50.486948Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518775024956729:2067] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:50.493914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518779319924087:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53590
2025-11-11T17:19:50.517645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518778342476429:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.471825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:50.474520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518774832077992:2098] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.521935Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518779319924030:2069] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:50.513996Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518772307615974:2068] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.471876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518771595033478:2080] [node 4] ICP22 created new session: [2:7571518775890000815:2080]
No RDMA context found for GID: 2025-11-11T17:19:50.525647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518774130899400:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.535495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518778342476426:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.471917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518775890000815:2080] [node 4] ICS09 handshake done sender: [2:7571518775890000781:2098] self: [2:7571518775890000780:0] peer: [4:7571518776602583328:0] socket: 368 qp: -1
2025-11-11T17:19:50.471947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518775890000815:2080] [node 4] ICS10 traffic start
2025-11-11T17:19:50.471977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:50.509911Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518771595033481:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.509936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518771595033481:2096] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.474573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:50.514046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518772307615974:2068] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.526918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518772307615950:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518776602583294:2108] held: no
2025-11-11T17:19:50.577862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518776602583331:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.581927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518771595033478:2080] [node 4] ICP35 event processing took too much time PendingConnection:278(109889979ns) {HandleHandshakeStatus:329(109887767ns) {DropIncomingHandshake:441(167ns) DropOutgoingHandshake:456(314ns) SwitchToState:178(247ns) ProcessPendingSessionEvents:495(94482407ns) {ForwardSessionEventToSession:612(30507664ns) ForwardSessionEventToSession:612(12007435ns) ForwardSessionEventToSession:612(4317ns) ForwardSessionEventToSession:612(12033455ns) ForwardSessionEventToSession:612(5259ns) ForwardSessionEventToSession:612(2567ns) ForwardSessionEventToSession:612(5070861ns) ForwardSessionEventToSession:612(3853ns) ForwardSessionEventToSession:612(18888678ns) ForwardSessionEventToSession:612(7964218ns)}}}
2025-11-11T17:19:50.582197Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518775890000786:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.582216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518775890000786:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.583368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518775890000817:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3646
2025-11-11T17:19:50.586196Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518775890000786:2099] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:50.586512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518771595033476:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.586559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:50.586607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518771595033476:2076] [node 5] ICP22 created new session: [2:7571518775890000819:2076]
2025-11-11T17:19:50.586651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518775890000819:2076] [node 5] ICS09 handshake done sender: [2:7571518775890000786:2099] self: [2:7571518775890000785:0] peer: [5:7571518778425866778:0] socket: 381 qp: -1
2025-11-11T17:19:50.586680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518775890000819:2076] [node 5] ICS10 traffic start
2025-11-11T17:19:50.586714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:50.474620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518774832077992:2098] [node 4] ICP22 created new session: [1:7571518779127045323:2098]
2025-11-11T17:19:50.474666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518779127045323:2098] [node 4] ICS09 handshake done sender: [1:7571518779127045296:2110] self: [1:7571518779127045295:0] peer: [4:7571518776602583329:0] socket: 361 qp: -1
2025-11-11T17:19:50.474693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518779127045323:2098] [node 4] ICS10 traffic start
2025-11-11T17:19:50.474720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:50.499653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518779127045297:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.499676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518779127045297:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.500614Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518779127045297:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:50.500838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518774832077991:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.500876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:50.500919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518774832077991:2094] [node 6] ICP22 created new session: [1:7571518779127045326:2094]
2025-11-11T17:19:50.500965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518779127045326:2094] [node 6] ICS09 handshake done sender: [1:7571518779127045297:2111] self: [1:7571518779127045294:0] peer: [6:7571518778342476433:0] socket: 371 qp: -1
2025-11-11T17:19:50.500995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518779127045326:2094] [node 6] ICS10 traffic start
2025-11-11T17:19:50.501025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:50.508799Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:50.527174Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518779127045299:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.533753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518774130899400:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(27875566ns) {IncomingHandshake:261(27873180ns) {DropIncomingHandshake:441(1844ns) PrepareNewSessionHandshake:476(3969ns) {SwitchToState:178(3136ns)} GenerateSessionVirtualId:988(1124ns) IssueIncomingHandshakeReply:193(19932948ns) {SwitchToState:178(3457ns)}}}
2025-11-11T17:19:50.558635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518778425866769:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.558675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518778425866769:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.562786Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518778425866769:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:50.564791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866780:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
2025-11-11T17:19:50.564848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866781:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
2025-11-11T17:19:50.566533Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518778425866780:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.567769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518774130899401:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:50.570105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866784:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
2025-11-11T17:19:50.578746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518778425866764:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.593740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:50.593806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518778425866764:2069] [node 6] ICP22 created new session: [5:7571518778425866785:2069]
2025-11-11T17:19:50.598834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518778425866785:2069] [node 6] ICS09 handshake done sender: [5:7571518778425866769:2115] self: [5:7571518778425866768:0] peer: [6:7571518778342476438:0] socket: 389 qp: -1
2025-11-11T17:19:50.598880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518778425866785:2069] [node 6] ICS10 traffic start
2025-11-11T17:19:50.598921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:50.542383Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518779319924032:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.606490Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518778425866780:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:50.592683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518776602583330:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.610344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518776602583332:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-11T17:19:50.611351Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518772307615976:2069] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.611379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518772307615976:2069] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.612190Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518772307615982:2071] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:50.612460Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518776602583297:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:50.612648Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518776602583294:2108] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:50.612923Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518772307615974:2068] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:50.527212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518779127045299:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.562440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476439:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.594539Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518775890000812:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.562880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476440:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.570198Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22225) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:50.585791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518774047509042:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22225) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518778342476394:2122] held: no
2025-11-11T17:19:50.585942Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518778342476394:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:50.594289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518774047509042:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.594333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:50.594383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518774047509042:2050] [node 3] ICP22 created new session: [6:7571518778342476441:2050]
2025-11-11T17:19:50.594429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518778342476441:2050] [node 3] ICS09 handshake done sender: [6:7571518778342476394:2122] self: [6:7571518778342476432:0] peer: [3:7571518779319924031:0] socket: 370 qp: -1
2025-11-11T17:19:50.594462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518778342476441:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:50.594498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:50.601943Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518778342476429:2130] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:50.618145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518778342476426:2109] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.618220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:50.618281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518778342476426:2109] [node 5] ICP22 created new session: [6:7571518778342476444:2109]
2025-11-11T17:19:50.618333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518778342476444:2109] [node 5] ICS09 handshake done sender: [6:7571518778342476429:2130] self: [6:7571518778342476438:0] peer: [5:7571518778425866768:0] socket: 392 qp: -1
2025-11-11T17:19:50.618370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518778342476444:2109] [node 5] ICS10 traffic start
2025-11-11T17:19:50.618406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:50.630716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518774130899401:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.630803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:50.630850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518774130899401:2050] [node 3] ICP22 created new session: [5:7571518778425866789:2050]
2025-11-11T17:19:50.630894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518778425866789:2050] [node 3] ICS09 handshake done sender: [5:7571518778425866780:2118] self: [5:7571518778425866783:0] peer: [3:7571518779319924085:0] socket: 398 qp: -1
2025-11-11T17:19:50.630930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518778425866789:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:50.630961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:50.631534Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22225) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:50.613266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518772307615952:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.542419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518779319924032:2070] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.633165Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:50.571264Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518779127045299:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:50.598815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518771595033481:2096] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:50.542989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518775024956726:2062] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.613338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:50.605663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518774832077997:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.615065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:50.619879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518774832077997:2096] [node 5] ICP22 created new session: [1:7571518779127045329:2096]
2025-11-11T17:19:50.645806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518779127045329:2096] [node 5] ICS09 handshake done sender: [1:7571518779127045299:2112] self: [1:7571518779127045298:0] peer: [5:7571518778425866777:0] socket: 357 qp: -1
2025-11-11T17:19:50.653771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518779127045329:2096] [node 5] ICS10 traffic start
2025-11-11T17:19:50.653835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:50.654291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518774832077997:2096] [node 5] ICP35 event processing took too much time PendingConnection:278(48437509ns) {HandleHandshakeStatus:329(48435313ns) {DropIncomingHandshake:441(15338ns) DropOutgoingHandshake:456(471ns) SwitchToState:178(590ns) ProcessPendingSessionEvents:495(107219ns) {ForwardSessionEventToSession:612(55511ns) ForwardSessionEventToSession:612(6422ns) ForwardSessionEventToSession:612(5055ns) ForwardSessionEventToSession:612(3743ns) ForwardSessionEventToSession:612(8256ns) ForwardSessionEventToSession:612(3103ns) ForwardSessionEventToSession:612(4356ns) ForwardSessionEventToSession:612(4744ns) ForwardSessionEventToSession:612(2158ns) ForwardSessionEventToSession:612(5090ns)}}}
2025-11-11T17:19:50.598999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518771595033474:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.606740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:50.620036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518771595033474:2072] [node 6] ICP22 created new session: [2:7571518775890000823:2072]
2025-11-11T17:19:50.637890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518775890000817:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.638003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518771595033476:2076] [node 5] ICP18 handshake (actor [2:7571518775890000817:2119]) is too old
2025-11-11T17:19:50.645834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518775890000823:2072] [node 6] ICS09 handshake done sender: [2:7571518771595033481:2096] self: [2:7571518771595033480:0] peer: [6:7571518778342476431:0] socket: 351 qp: -1
2025-11-11T17:19:50.650014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518775890000823:2072] [node 6] ICS10 traffic start
2025-11-11T17:19:50.650065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:50.613373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518772307615952:2050] [node 3] ICP22 created new session: [4:7571518776602583337:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.660032Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:50.543048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:50.665987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518778425866767:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:50.613418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518776602583337:2050] [node 3] ICS09 handshake done sender: [4:7571518772307615982:2071] self: [4:7571518776602583281:0] peer: [3:7571518775024956728:0] socket: 349 qp: -1
2025-11-11T17:19:50.613454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518776602583337:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:50.613485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:50.655029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518772307615950:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.658549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518772307615951:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.658609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:50.658661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518772307615951:2049] [node 2] ICP22 created new session: [4:7571518776602583339:2049]
2025-11-11T17:19:50.658711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518776602583339:2049] [node 2] ICS09 handshake done sender: [4:7571518776602583297:2117] self: [4:7571518776602583328:0] peer: [2:7571518775890000780:0] socket: 369 qp: -1
2025-11-11T17:19:50.658748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518776602583339:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:50.658781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:50.659303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518772307615971:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.659364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:50.659405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518772307615971:2066] [node 5] ICP22 created new session: [4:7571518776602583341:2066]
2025-11-11T17:19:50.661192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518776602583341:2066] [node 5] ICS09 handshake done sender: [4:7571518772307615974:2068] self: [4:7571518772307615973:0] peer: [5:7571518778425866776:0] socket: 342 qp: -1
2025-11-11T17:19:50.661233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518776602583341:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:50.661268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:50.674417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518776602583334:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-11T17:19:50.681749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:50.681822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518772307615950:2048] [node 1] ICP22 created new session: [4:7571518776602583343:2048]
2025-11-11T17:19:50.681866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518776602583343:2048] [node 1] ICS09 handshake done sender: [4:7571518776602583294:2108] self: [4:7571518776602583329:0] peer: [1:7571518779127045295:0] socket: 367 qp: -1
2025-11-11T17:19:50.681906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518776602583343:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:50.681942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:50.682167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518772307615950:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32165372ns) {HandleHandshakeStatus:329(32162766ns) {DropIncomingHandshake:441(2314ns) DropOutgoingHandshake:456(1858ns) SwitchToState:178(1643ns) ProcessPendingSessionEvents:495(1564ns)}}
2025-11-11T17:19:50.682344Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518772307615976:2069] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:50.685368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518772307615972:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.685472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:50.685523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518772307615972:2067] [node 6] ICP22 created new session: [4:7571518776602583345:2067]
2025-11-11T17:19:50.685563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518776602583345:2067] [node 6] ICS09 handshake done sender: [4:7571518772307615976:2069] self: [4:7571518772307615975:0] peer: [6:7571518778342476430:0] socket: 343 qp: -1
2025-11-11T17:19:50.685622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518776602583345:2067] [node 6] ICS10 traffic start
2025-11-11T17:19:50.686053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:50.697786Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518774130899401:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:50.697919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518774130899399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518778425866767:2114] held: no
2025-11-11T17:19:50.698025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518774130899399:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.698095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:50.698138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518774130899399:2048] [node 1] ICP22 created new session: [5:7571518778425866792:2048]
2025-11-11T17:19:50.698183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518778425866792:2048] [node 1] ICS09 handshake done sender: [5:7571518778425866767:2114] self: [5:7571518778425866777:0] peer: [1:7571518779127045298:0] socket: 387 qp: -1
2025-11-11T17:19:50.698218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518778425866792:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:50.698251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:50.698728Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:50.543100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518775024956726:2062] [node 4] ICP22 created new session: [3:7571518779319924090:2062]
2025-11-11T17:19:50.682745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518771595033474:2072] [node 6] ICP35 event processing took too much time PendingConnection:278(83527534ns) {HandleHandshakeStatus:329(83526090ns) {DropIncomingHandshake:441(10286ns) DropOutgoingHandshake:456(507ns) SwitchToState:178(206ns) ProcessPendingSessionEvents:495(8687859ns) {ForwardSessionEventToSession:612(8631455ns) ForwardSessionEventToSession:612(8014ns) ForwardSessionEventToSession:612(3965ns) ForwardSessionEventToSession:612(6717ns) ForwardSessionEventToSession:612(3222ns) ForwardSessionEventToSession:612(5575ns) ForwardSessionEventToSession:612(2122ns) ForwardSessionEventToSession:612(2498ns) ForwardSessionEventToSession:612(7316ns) ForwardSessionEventToSession:612(3027ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.721829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518778425866764:2069] [node 6] ICP35 event processing took too much time PendingConnection:278(112454006ns) {HandleHandshakeStatus:329(99387299ns) {DropIncomingHandshake:441(298ns) DropOutgoingHandshake:456(523ns) SwitchToState:178(461ns) ProcessPendingSessionEvents:495(64164973ns) {ForwardSessionEventToSession:612(44540740ns) ForwardSessionEventToSession:612(3307813ns) ForwardSessionEventToSession:612(3045ns) ForwardSessionEventToSession:612(7949449ns) ForwardSessionEventToSession:612(6826ns) ForwardSessionEventToSession:612(4021464ns) ForwardSessionEventToSession:612(565710ns) ForwardSessionEventToSession:612(5517ns) ForwardSessionEventToSession:612(3037ns) ForwardSessionEventToSession:612(3735069ns)}}}
2025-11-11T17:19:50.722094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866787:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.543155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518779319924090:2062] [node 4] ICS09 handshake done sender: [3:7571518775024956729:2067] self: [3:7571518775024956728:0] peer: [4:7571518776602583281:0] socket: 347 qp: -1
2025-11-11T17:19:50.543196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518779319924090:2062] [node 4] ICS10 traffic start
2025-11-11T17:19:50.543236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:50.543954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518775024956707:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.543996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:50.544037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518775024956707:2049] [node 2] ICP22 created new session: [3:7571518779319924092:2049]
2025-11-11T17:19:50.544080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518779319924092:2049] [node 2] ICS09 handshake done sender: [3:7571518779319924030:2069] self: [3:7571518779319924035:0] peer: [2:7571518771595033482:0] socket: 353 qp: -1
2025-11-11T17:19:50.544109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518779319924092:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:50.544184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:50.561938Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518779319924032:2070] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:50.563221Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518779319924087:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.563266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518775024956727:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.563328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:50.563371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518775024956727:2064] [node 6] ICP22 created new session: [3:7571518779319924095:2064]
2025-11-11T17:19:50.563418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518779319924095:2064] [node 6] ICS09 handshake done sender: [3:7571518779319924032:2070] self: [3:7571518779319924031:0] peer: [6:7571518778342476432:0] socket: 363 qp: -1
2025-11-11T17:19:50.563451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518779319924095:2064] [node 6] ICS10 traffic start
2025-11-11T17:19:50.563488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:50.564004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518775024956727:2064] [node 6] ICP18 handshake (actor [3:7571518779319924087:2118]) is too old
2025-11-11T17:19:50.566365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518779319924097:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53590
2025-11-11T17:19:50.568109Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518779319924086:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.568128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518779319924086:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:50.568872Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518779319924086:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:50.569066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518779319924077:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:50.569104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:50.569142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518779319924077:2075] [node 5] ICP22 created new session: [3:7571518779319924099:2075]
2025-11-11T17:19:50.569184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518779319924099:2075] [node 5] ICS09 handshake done sender: [3:7571518779319924086:2117] self: [3:7571518779319924085:0] peer: [5:7571518778425866783:0] socket: 379 qp: -1
2025-11-11T17:19:50.569211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518779319924099:2075] [node 5] ICS10 traffic start
2025-11-11T17:19:50.569258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:50.578243Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518779319924097:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.581735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518779319924077:2075] [node 5] ICP18 handshake (actor [3:7571518779319924097:2119]) is too old
2025-11-11T17:19:50.684341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518779319924101:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53590
2025-11-11T17:19:50.685820Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.749751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518774047509041:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518778342476393:2121] held: no
2025-11-11T17:19:50.750059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518774047509041:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24097210ns) {HandleHandshakeStatus:395(24093633ns) {LogHandshakeFail:472(14443ns) {UpdateErrorStateLog:544(5789ns)} DropOutgoingHandshake:456(2179ns) ProcessPendingSessionEvents:495(57525ns) {DropSessionEvent:512(11108ns) DropSessionEvent:512(6537ns) DropSessionEvent:512(3152ns) DropSessionEvent:512(5557ns) DropSessionEvent:512(3277ns) DropSessionEvent:512(4376ns) DropSessionEvent:512(3658ns) DropSessionEvent:512(3326ns) DropSessionEvent:512(3180ns) DropSessionEvent:512(9370ns)}}}
2025-11-11T17:19:50.750695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476446:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.729826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866788:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
2025-11-11T17:19:50.729986Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:50.742379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518774130899400:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518778425866766:2113] held: no
2025-11-11T17:19:50.742832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518778425866764:2069] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:50.743104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518778425866791:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15984
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.753991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518776602583334:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:50.761772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518772307615972:2067] [node 6] ICP18 handshake (actor [4:7571518776602583334:2121]) is too old
2025-11-11T17:19:50.753868Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518778342476393:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:50.758036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518778342476447:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53610
2025-11-11T17:19:50.766339Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518778342476365:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:50.768422Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25847) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:50.766538Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518778425866766:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:50.770717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518776602583347:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-11T17:19:50.767301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518774130899400:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.767370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:50.767419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518774130899400:2049] [node 2] ICP22 created new session: [5:7571518778425866795:2049]
2025-11-11T17:19:50.767467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518778425866795:2049] [node 2] ICS09 handshake done sender: [5:7571518778425866766:2113] self: [5:7571518778425866778:0] peer: [2:7571518775890000785:0] socket: 386 qp: -1
2025-11-11T17:19:50.767496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518778425866795:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:50.767526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:50.767971Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518778425866710:2060] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:50.769392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518778342476412:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.769469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:50.769518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518778342476412:2107] [node 4] ICP22 created new session: [6:7571518778342476448:2107]
2025-11-11T17:19:50.769557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518778342476448:2107] [node 4] ICS09 handshake done sender: [6:7571518778342476365:2112] self: [6:7571518778342476430:0] peer: [4:7571518772307615975:0] socket: 346 qp: -1
2025-11-11T17:19:50.769601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518778342476448:2107] [node 4] ICS10 traffic start
2025-11-11T17:19:50.769635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:50.769829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518774047509041:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.769870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:50.769912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518774047509041:2049] [node 2] ICP22 created new session: [6:7571518778342476450:2049]
2025-11-11T17:19:50.769957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518778342476450:2049] [node 2] ICS09 handshake done sender: [6:7571518778342476393:2121] self: [6:7571518778342476431:0] peer: [2:7571518771595033480:0] socket: 356 qp: -1
2025-11-11T17:19:50.769987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518778342476450:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:50.770012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:50.786408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518778425866765:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:50.786469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:50.786523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518778425866765:2111] [node 4] ICP22 created new session: [5:7571518778425866797:2111]
2025-11-11T17:19:50.786570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518778425866797:2111] [node 4] ICS09 handshake done sender: [5:7571518778425866710:2060] self: [5:7571518778425866776:0] peer: [4:7571518772307615973:0] socket: 345 qp: -1
2025-11-11T17:19:50.786601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518778425866797:2111] [node 4] ICS10 traffic start
2025-11-11T17:19:50.786635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:50.788481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518778342476412:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(18883861ns) {HandleHandshakeStatus:329(18882300ns) {DropIncomingHandshake:441(685ns) DropOutgoingHandshake:456(7130ns) SwitchToState:178(790ns) ProcessPendingSessionEvents:495(88440ns) {ForwardSessionEventToSession:612(50578ns) ForwardSessionEventToSession:612(2787ns) ForwardSessionEventToSession:612(4201ns) ForwardSessionEventToSession:612(3129ns) ForwardSessionEventToSession:612(2005ns) ForwardSessionEventToSession:612(4892ns) ForwardSessionEventToSession:612(2354ns) ForwardSessionEventToSession:612(4549ns) ForwardSessionEventToSession:612(4594ns) ForwardSessionEventToSession:612(2532ns)}}}
2025-11-11T17:19:50.788561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518778342476412:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:50.789832Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:50.846524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:50.846759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518775890000810:2048] [node 1] ICS01 socket: 364 reason# EndOfStream
2025-11-11T17:19:50.846796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518771595033427:2048] [node 1] ICP30 unregister session Session# [2:7571518775890000810:2048] VirtualId# [2:7571518775890000797:0]
2025-11-11T17:19:50.846829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518775890000810:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.846899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:50.846935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:50.857740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:50.857966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518779319924038:2048] [node 1] ICS01 socket: 362 reason# EndOfStream
2025-11-11T17:19:50.858010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518775024956706:2048] [node 1] ICP30 unregister session Session# [3:7571518779319924038:2048] VirtualId# [3:7571518779319924036:0]
2025-11-11T17:19:50.858045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518779319924038:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.858152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:50.858198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:50.865738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:50.865978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518778425866792:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-11-11T17:19:50.866015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518774130899399:2048] [node 1] ICP30 unregister session Session# [5:7571518778425866792:2048] VirtualId# [5:7571518778425866777:0]
2025-11-11T17:19:50.866043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518778425866792:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.866121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:50.866158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:50.865748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:50.865976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518778342476435:2048] [node 1] ICS01 socket: 375 reason# EndOfStream
2025-11-11T17:19:50.866012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518774047509040:2048] [node 1] ICP30 unregister session Session# [6:7571518778342476435:2048] VirtualId# [6:7571518778342476433:0]
2025-11-11T17:19:50.866047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518778342476435:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.866121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:50.866158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:50.866232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:50.866430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518776602583343:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-11T17:19:50.866463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518772307615950:2048] [node 1] ICP30 unregister session Session# [4:7571518776602583343:2048] VirtualId# [4:7571518776602583329:0]
2025-11-11T17:19:50.866484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518776602583343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.866532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:50.866563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:50.937943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:50.938160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518779319924092:2049] [node 2] ICS01 socket: 353 reason# EndOfStream
2025-11-11T17:19:50.938200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518775024956707:2049] [node 2] ICP30 unregister session Session# [3:7571518779319924092:2049] VirtualId# [3:7571518779319924035:0]
2025-11-11T17:19:50.938237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518779319924092:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.938348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:50.938392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:50.952891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:50.953135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518776602583339:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-11T17:19:50.953180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518772307615951:2049] [node 2] ICP30 unregister session Session# [4:7571518776602583339:2049] VirtualId# [4:7571518776602583328:0]
2025-11-11T17:19:50.953214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518776602583339:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.953309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:50.953358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:50.953790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:50.953967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518778342476450:2049] [node 2] ICS01 socket: 356 reason# EndOfStream
2025-11-11T17:19:50.953996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518774047509041:2049] [node 2] ICP30 unregister session Session# [6:7571518778342476450:2049] VirtualId# [6:7571518778342476431:0]
2025-11-11T17:19:50.954021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518778342476450:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.954085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:50.954120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:50.957703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:50.957904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518778425866795:2049] [node 2] ICS01 socket: 386 reason# EndOfStream
2025-11-11T17:19:50.958009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518774130899400:2049] [node 2] ICP30 unregister session Session# [5:7571518778425866795:2049] VirtualId# [5:7571518778425866778:0]
2025-11-11T17:19:50.958043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518778425866795:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:50.959745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:50.959840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:51.077777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:51.077985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518778342476441:2050] [node 3] ICS01 socket: 370 reason# EndOfStream
2025-11-11T17:19:51.078020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518774047509042:2050] [node 3] ICP30 unregister session Session# [6:7571518778342476441:2050] VirtualId# [6:7571518778342476432:0]
2025-11-11T17:19:51.078052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518778342476441:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:51.078125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:51.078164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:51.078243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:51.078420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518778425866789:2050] [node 3] ICS01 socket: 398 reason# EndOfStream
2025-11-11T17:19:51.078451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518774130899401:2050] [node 3] ICP30 unregister session Session# [5:7571518778425866789:2050] VirtualId# [5:7571518778425866783:0]
2025-11-11T17:19:51.078474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518778425866789:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:51.078515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:51.078546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:51.081701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:51.081894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518776602583337:2050] [node 3] ICS07 socket disconnect 349 reason# EndOfStream
2025-11-11T17:19:51.081922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518776602583337:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:51.082005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:51.082044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518776602583337:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:51.089947Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22225) Couldn't connect to any resolved address
2025-11-11T17:19:51.090254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518772307615952:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22225) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:51.090314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518776602583337:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:51.090347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518772307615952:2050] [node 3] ICP30 unregister session Session# [4:7571518776602583337:2050] VirtualId# [4:7571518776602583281:0]
2025-11-11T17:19:51.090398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:51.090431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518772307615952:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22225) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:51.090462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22225) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:51.105793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518772307615952:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:51.198750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:51.199036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518778425866797:2111] [node 4] ICS01 socket: 345 reason# EndOfStream
2025-11-11T17:19:51.199102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518778425866765:2111] [node 4] ICP30 unregister session Session# [5:7571518778425866797:2111] VirtualId# [5:7571518778425866776:0]
2025-11-11T17:19:51.199162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518778425866797:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:51.199252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:51.199308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:51.199470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:51.199781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518778342476448:2107] [node 4] ICS01 socket: 346 reason# EndOfStream
2025-11-11T17:19:51.199830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518778342476412:2107] [node 4] ICP30 unregister session Session# [6:7571518778342476448:2107] VirtualId# [6:7571518778342476430:0]
2025-11-11T17:19:51.199874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518778342476448:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:51.199955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:51.200021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:51.285733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:51.285980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518778342476444:2109] [node 5] ICS01 socket: 392 reason# EndOfStream
2025-11-11T17:19:51.286024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518778342476426:2109] [node 5] ICP30 unregister session Session# [6:7571518778342476444:2109] VirtualId# [6:7571518778342476438:0]
2025-11-11T17:19:51.286060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518778342476444:2109] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:51.286164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:51.286205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:51.486662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518782136869694:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:51.486704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518782136869695:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:51.535684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518782867080205:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:51.535724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518782867080206:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:51.593379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518781475421454:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:51.593463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518781475421455:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:51.659359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518782783690305:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:51.659402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518782783690306:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:51.659430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518782783690307:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:51.709510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518783535353523:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:51.709550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518783535353524:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:51.709581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518783535353525:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:51.950244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518781475421468:2060] [node 4] ICP01 ready to work
2025-11-11T17:19:52.047423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518785770388782:2065] [node 5] ICP01 ready to work
2025-11-11T17:19:52.005671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518782867080235:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:51.902011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518783657127244:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:51.902053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518783657127245:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:51.902082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518783657127246:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:52.053757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518783657127261:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:52.047549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518787162047533:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:52.053669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518786431837012:2064] [node 5] ICP01 ready to work
2025-11-11T17:19:52.095231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518785770388783:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:52.097751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518786431837014:2062] [node 6] ICP01 ready to work
2025-11-11T17:19:52.086735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518787162047533:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(37113549ns) {Configure:104(37110140ns) {ProcessConfigured:135(65206ns) {StartInitialHandshake:157(62150ns) {DropHandshakes:469(2385ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(397ns)} GenerateSessionVirtualId:988(871ns) PrepareNewSessionHandshake:476(1191ns) {SwitchToState:178(355ns)}}}}}
2025-11-11T17:19:52.162150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518787078657634:2066] [node 6] ICP01 ready to work
2025-11-11T17:19:52.227340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518782783690306:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(282743487ns) {Configure:104(282739085ns) {ProcessConfigured:135(76367ns) {StartInitialHandshake:157(73403ns) {DropHandshakes:469(2245ns) {DropIncomingHandshake:441(524ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:476(1820ns) {SwitchToState:178(813ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.230836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657640:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
2025-11-11T17:19:52.231033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657641:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
2025-11-11T17:19:52.101382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518785770388788:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59661
2025-11-11T17:19:52.119663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518787162047537:2072] [node 6] ICP01 ready to work
2025-11-11T17:19:52.129855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518786431837025:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.257784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518787078657641:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.267278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518787830320847:2060] [node 4] ICP01 ready to work
2025-11-11T17:19:52.342968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518782136869694:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(247715694ns) {Configure:104(247705978ns) {ProcessConfigured:135(69126ns) {StartInitialHandshake:157(66094ns) {DropHandshakes:469(2182ns) {DropIncomingHandshake:441(475ns) DropOutgoingHandshake:456(357ns)} GenerateSessionVirtualId:988(1122ns) PrepareNewSessionHandshake:476(2598ns) {SwitchToState:178(1172ns)}}}}}
2025-11-11T17:19:52.257774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518783657127291:2063] [node 5] ICP01 ready to work
2025-11-11T17:19:52.241644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518785770388788:2080] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.238273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518787162047537:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(101543649ns) {Configure:104(101538950ns) {ProcessConfigured:135(31993576ns) {StartInitialHandshake:157(31991393ns) {DropHandshakes:469(2644ns) {DropIncomingHandshake:441(980ns) DropOutgoingHandshake:456(549ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:476(2887ns) {SwitchToState:178(146ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.264949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518787078657652:2069] [node 5] ICP01 ready to work
2025-11-11T17:19:52.315758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094590:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.247598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518781475421454:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(152094259ns) {Configure:104(152090507ns) {ProcessConfigured:135(69143ns) {StartInitialHandshake:157(66154ns) {DropHandshakes:469(2605ns) {DropIncomingHandshake:441(698ns) DropOutgoingHandshake:456(394ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:476(1350ns) {SwitchToState:178(393ns)}}}}}
2025-11-11T17:19:52.378866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518785770388828:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59661
2025-11-11T17:19:52.384323Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518785770388790:2081] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.384360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518785770388790:2081] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.386862Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518785770388790:2081] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:52.365981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518787162047582:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20262
2025-11-11T17:19:52.376996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518783535353524:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(317978228ns) {Configure:104(317973662ns) {ProcessConfigured:135(14522563ns) {StartInitialHandshake:157(14518369ns) {DropHandshakes:469(810ns) {DropIncomingHandshake:441(222ns) DropOutgoingHandshake:456(150ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:476(2461ns) {SwitchToState:178(141ns)}}}}}
2025-11-11T17:19:52.409146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518785770388783:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.410730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:52.317347Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518787078657640:2083] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.385337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518787162047583:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20262
2025-11-11T17:19:52.369922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518783657127246:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(428025577ns) {Configure:104(428016714ns) {ProcessConfigured:135(19716929ns) {StartInitialHandshake:157(19713875ns) {DropHandshakes:469(2519ns) {DropIncomingHandshake:441(1216ns) DropOutgoingHandshake:456(315ns)} GenerateSessionVirtualId:988(842ns) PrepareNewSessionHandshake:476(4876ns) {SwitchToState:178(2145ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.434244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518782136869695:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(460303564ns) {Configure:104(460297043ns) {ProcessConfigured:135(50463ns) {StartInitialHandshake:157(47149ns) {DropHandshakes:469(2277ns) {DropIncomingHandshake:441(469ns) DropOutgoingHandshake:456(765ns)} GenerateSessionVirtualId:988(867ns) PrepareNewSessionHandshake:476(977ns) {SwitchToState:178(336ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.435394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518786431837064:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54563
2025-11-11T17:19:52.435528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518786431837065:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54563
2025-11-11T17:19:52.401734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518782783690306:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.414258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518787162047582:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.415229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518785770388783:2063] [node 6] ICP22 created new session: [3:7571518785770388830:2063]
2025-11-11T17:19:52.426891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518787830320847:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(49425827ns) {Configure:104(49421939ns) {ProcessConfigured:135(65324ns) {StartInitialHandshake:157(62201ns) {DropHandshakes:469(5048ns) {DropIncomingHandshake:441(513ns) DropOutgoingHandshake:456(417ns)} GenerateSessionVirtualId:988(1121ns) PrepareNewSessionHandshake:476(1335ns) {SwitchToState:178(432ns)}}}}}
2025-11-11T17:19:52.457683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518782783690306:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31456811ns) {IncomingHandshake:261(11544896ns) {DropIncomingHandshake:441(65ns) PrepareNewSessionHandshake:476(1462ns) {SwitchToState:178(657ns)} GenerateSessionVirtualId:988(195ns) IssueIncomingHandshakeReply:193(11518721ns) {SwitchToState:178(3893ns)}}}
2025-11-11T17:19:52.457976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518782783690305:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(462985092ns) {Configure:104(462978982ns) {ProcessConfigured:135(30601995ns) {StartInitialHandshake:157(30598593ns) {DropHandshakes:469(2110ns) {DropIncomingHandshake:441(436ns) DropOutgoingHandshake:456(436ns)} GenerateSessionVirtualId:988(985ns) PrepareNewSessionHandshake:476(4247ns) {SwitchToState:178(1984ns)}}}}}
2025-11-11T17:19:52.374226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518783657127244:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(137318510ns) {Configure:104(137315036ns) {ProcessConfigured:135(12106529ns) {StartInitialHandshake:157(12102779ns) {DropHandshakes:469(2259ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(390ns)} GenerateSessionVirtualId:988(1105ns) PrepareNewSessionHandshake:476(4331ns) {SwitchToState:178(1799ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.447062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518785770388830:2063] [node 6] ICS09 handshake done sender: [3:7571518785770388790:2081] self: [3:7571518785770388789:0] peer: [6:7571518787952094619:0] socket: 354 qp: -1
2025-11-11T17:19:52.470013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518787162047586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20262
2025-11-11T17:19:52.470094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518787162047588:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20262
2025-11-11T17:19:52.460741Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837064:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.489754Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518787162047541:2081] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.489793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518787162047541:2081] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.429257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320885:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.429434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320886:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.429532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320887:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.433822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518787830320888:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:52.454765Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518787830320887:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.465711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518787830320885:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.493739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518787830320886:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.374407Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518787952094590:2094] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.490914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518782783690307:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.454501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518785770388834:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59661
2025-11-11T17:19:52.376512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518783657127246:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.386098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094620:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.387477Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518787952094590:2094] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:52.387622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518783657127246:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.387694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:52.387738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518783657127246:2050] [node 3] ICP22 created new session: [6:7571518787952094621:2050]
2025-11-11T17:19:52.387785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518787952094621:2050] [node 3] ICS09 handshake done sender: [6:7571518787952094590:2094] self: [6:7571518787952094619:0] peer: [3:7571518785770388789:0] socket: 357 qp: -1
2025-11-11T17:19:52.387818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518787952094621:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:52.387853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:52.492314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094625:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.495847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518782867080205:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(393251012ns) {Configure:104(393244497ns) {ProcessConfigured:135(32059498ns) {StartInitialHandshake:157(32056236ns) {DropHandshakes:469(1919ns) {DropIncomingHandshake:441(436ns) DropOutgoingHandshake:456(568ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:476(4648ns) {SwitchToState:178(2087ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.544167Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518787952094625:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.544308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518783657127261:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.455016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518785770388830:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:52.513175Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837065:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.528102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518783535353524:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.495980Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518787162047541:2081] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:52.491351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657680:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
2025-11-11T17:19:52.461560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:52.476332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518781475421455:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(491404710ns) {Configure:104(483300672ns) {ProcessConfigured:135(27224823ns) {StartInitialHandshake:157(22393904ns) {DropHandshakes:469(2095ns) {DropIncomingHandshake:441(194ns) DropOutgoingHandshake:456(335ns)} GenerateSessionVirtualId:988(1203ns) PrepareNewSessionHandshake:476(4640ns) {SwitchToState:178(1663ns)}}}}}
2025-11-11T17:19:52.495564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518781475421455:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.518972Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518785770388834:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.526152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518785770388783:2063] [node 6] ICP35 event processing took too much time PendingConnection:278(116769195ns) {HandleHandshakeStatus:329(116764316ns) {DropIncomingHandshake:441(480ns) DropOutgoingHandshake:456(873ns) SwitchToState:178(523ns) ProcessPendingSessionEvents:495(12115077ns) {ForwardSessionEventToSession:612(12052481ns) ForwardSessionEventToSession:612(10140ns) ForwardSessionEventToSession:612(2862ns) ForwardSessionEventToSession:612(2336ns) ForwardSessionEventToSession:612(8489ns) ForwardSessionEventToSession:612(2881ns) ForwardSessionEventToSession:612(2869ns) ForwardSessionEventToSession:612(8730ns) ForwardSessionEventToSession:612(2268ns) ForwardSessionEventToSession:612(5049ns)}}}
2025-11-11T17:19:52.526784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518781475421454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.529938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518781475421470:2061] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.529973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518781475421470:2061] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.540461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518785770388839:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59661
2025-11-11T17:19:52.544565Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518781475421470:2061] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:52.544934Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518785770388834:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:52.545251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518781475421468:2060] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.545312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:52.545362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518781475421468:2060] [node 4] ICP22 created new session: [3:7571518785770388840:2060]
2025-11-11T17:19:52.545400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518785770388840:2060] [node 4] ICS09 handshake done sender: [3:7571518781475421470:2061] self: [3:7571518781475421469:0] peer: [4:7571518787078657683:0] socket: 348 qp: -1
2025-11-11T17:19:52.545429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518785770388840:2060] [node 4] ICS10 traffic start
2025-11-11T17:19:52.545464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:52.555520Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518785770388828:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.570235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518781475421468:2060] [node 4] ICP35 event processing took too much time PendingConnection:278(24792916ns) {HandleHandshakeStatus:329(24791073ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(526ns) SwitchToState:178(277ns) ProcessPendingSessionEvents:495(24467618ns) {ForwardSessionEventToSession:612(53218ns) ForwardSessionEventToSession:612(327107ns) ForwardSessionEventToSession:612(3881ns) ForwardSessionEventToSession:612(3243ns) ForwardSessionEventToSession:612(24042119ns) ForwardSessionEventToSession:612(5021ns) ForwardSessionEventToSession:612(7971ns) ForwardSessionEventToSession:612(5188ns) ForwardSessionEventToSession:612(2657ns) ForwardSessionEventToSession:612(5272ns)}}}
2025-11-11T17:19:52.570512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518781475421468:2060] [node 4] ICP18 handshake (actor [3:7571518785770388828:2115]) is too old
2025-11-11T17:19:52.571268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518781475421454:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.571313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:52.527089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837063:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.538722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518782867080235:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.538777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:52.492765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657681:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.492826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657682:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
2025-11-11T17:19:52.510431Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518787078657680:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.512883Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518787078657641:2084] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:52.538202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518782783690306:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.538252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:52.538301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518782783690306:2049] [node 2] ICP22 created new session: [4:7571518787078657685:2049]
2025-11-11T17:19:52.538347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518787078657685:2049] [node 2] ICS09 handshake done sender: [4:7571518787078657641:2084] self: [4:7571518787078657678:0] peer: [2:7571518787162047540:0] socket: 356 qp: -1
2025-11-11T17:19:52.538379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518787078657685:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:52.538410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:52.578032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518782783690306:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(39634513ns) {HandleHandshakeStatus:329(39632366ns) {DropIncomingHandshake:441(1100ns) DropOutgoingHandshake:456(4319ns) SwitchToState:178(476ns) ProcessPendingSessionEvents:495(39312483ns) {ForwardSessionEventToSession:612(42648ns) ForwardSessionEventToSession:612(11729357ns) ForwardSessionEventToSession:612(3362ns) ForwardSessionEventToSession:612(13850018ns) ForwardSessionEventToSession:612(10113ns) ForwardSessionEventToSession:612(2960ns) ForwardSessionEventToSession:612(6270249ns) ForwardSessionEventToSession:612(3398ns) ForwardSessionEventToSession:612(2902ns) ForwardSessionEventToSession:612(7978ns)}}}
2025-11-11T17:19:52.578233Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518787078657643:2085] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.578254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518787078657643:2085] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.579525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657689:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
2025-11-11T17:19:52.588603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518787078657652:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(151355422ns) {Configure:104(151349985ns) {ProcessConfigured:135(31781610ns) {StartInitialHandshake:157(31779446ns) {DropHandshakes:469(1854ns) {DropIncomingHandshake:441(542ns) DropOutgoingHandshake:456(343ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:476(3925ns) {SwitchToState:178(2074ns)}}}}}
2025-11-11T17:19:52.590957Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3561) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.598344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094624:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.598773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094627:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.571413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518783535353525:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(470139994ns) {Configure:104(470135491ns) {ProcessConfigured:135(39721485ns) {StartInitialHandshake:157(39718587ns) {DropHandshakes:469(2141ns) {DropIncomingHandshake:441(436ns) DropOutgoingHandshake:456(417ns)} GenerateSessionVirtualId:988(12103ns) PrepareNewSessionHandshake:476(3030ns) {SwitchToState:178(1112ns)}}}}}
2025-11-11T17:19:52.605968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518782783690307:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3561) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518787078657640:2083] held: no
2025-11-11T17:19:52.606173Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518787078657643:2085] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:52.538823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518782867080235:2065] [node 4] ICP22 created new session: [2:7571518787162047590:2065]
2025-11-11T17:19:52.527110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518786431837063:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.534071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518786431837063:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:52.553907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518786431837012:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(331245614ns) {Configure:104(331238871ns) {ProcessConfigured:135(15946429ns) {StartInitialHandshake:157(15943131ns) {DropHandshakes:469(2111ns) {DropIncomingHandshake:441(489ns) DropOutgoingHandshake:456(418ns)} GenerateSessionVirtualId:988(1624ns) PrepareNewSessionHandshake:476(3791ns) {SwitchToState:178(325ns)}}}}}
2025-11-11T17:19:52.610260Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518787952094624:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.610386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518783657127244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.610968Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518787952094625:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:52.611252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518783657127261:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.611323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:52.611369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518783657127261:2061] [node 4] ICP22 created new session: [6:7571518787952094629:2061]
2025-11-11T17:19:52.611412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518787952094629:2061] [node 4] ICS09 handshake done sender: [6:7571518787952094625:2118] self: [6:7571518787952094626:0] peer: [4:7571518787078657642:0] socket: 386 qp: -1
2025-11-11T17:19:52.611444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518787952094629:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:52.611480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:52.581408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518781475421454:2048] [node 1] ICP22 created new session: [3:7571518785770388843:2048]
2025-11-11T17:19:52.606481Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518787078657640:2083] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:52.593049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518783535353525:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.538863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518787162047590:2065] [node 4] ICS09 handshake done sender: [2:7571518787162047541:2081] self: [2:7571518787162047540:0] peer: [4:7571518787078657678:0] socket: 351 qp: -1
2025-11-11T17:19:52.589771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518785770388843:2048] [node 1] ICS09 handshake done sender: [3:7571518785770388834:2117] self: [3:7571518785770388837:0] peer: [1:7571518786431837062:0] socket: 384 qp: -1
2025-11-11T17:19:52.617825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518786431837072:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54563
2025-11-11T17:19:52.625701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837059:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.625727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518786431837059:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.626271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518782136869695:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.626321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:52.626368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518782136869695:2049] [node 3] ICP22 created new session: [1:7571518786431837076:2049]
2025-11-11T17:19:52.626413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518786431837076:2049] [node 3] ICS09 handshake done sender: [1:7571518786431837063:2113] self: [1:7571518786431837062:0] peer: [3:7571518785770388837:0] socket: 381 qp: -1
2025-11-11T17:19:52.626443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518786431837076:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:52.626475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:52.634291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518786431837075:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54563
2025-11-11T17:19:52.634881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518786431837059:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:52.638205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518786431837014:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.638257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:52.653765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518786431837014:2062] [node 6] ICP22 created new session: [1:7571518786431837080:2062]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.635918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518782783690307:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.674944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320903:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.676002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320904:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.676127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320905:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.589823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518785770388843:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:52.663139Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837072:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.668552Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837075:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.673560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518786431837080:2062] [node 6] ICS09 handshake done sender: [1:7571518786431837059:2111] self: [1:7571518786431837058:0] peer: [6:7571518787952094628:0] socket: 380 qp: -1
2025-11-11T17:19:52.685846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518786431837080:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:52.693598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:52.693676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837057:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.693710Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518786431837057:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.635992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:52.538891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518787162047590:2065] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.702464Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518786431837057:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:52.702770Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837061:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.702790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518786431837061:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.703386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518782136869694:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.703423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:52.703472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518782136869694:2048] [node 2] ICP22 created new session: [1:7571518786431837086:2048]
2025-11-11T17:19:52.703511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518786431837086:2048] [node 2] ICS09 handshake done sender: [1:7571518786431837057:2110] self: [1:7571518786431837056:0] peer: [2:7571518787162047596:0] socket: 379 qp: -1
2025-11-11T17:19:52.703536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518786431837086:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:52.703591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:52.706156Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518786431837061:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:52.589869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.693840Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:52.538918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:52.706372Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518787830320904:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.636036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518782783690307:2050] [node 3] ICP22 created new session: [4:7571518787078657692:2050]
2025-11-11T17:19:52.571146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518787162047535:2078] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.571178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518787162047535:2078] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.574706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518787162047588:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.580244Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518787162047539:2080] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.580261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518787162047539:2080] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.599554Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:52.599801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518782867080205:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.633739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518782867080235:2065] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:52.643829Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518787162047586:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.644289Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518787162047535:2078] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:52.644616Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518787162047539:2080] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:52.644841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518782867080206:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.644885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:52.644927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518782867080206:2049] [node 3] ICP22 created new session: [2:7571518787162047598:2049]
2025-11-11T17:19:52.644968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518787162047598:2049] [node 3] ICS09 handshake done sender: [2:7571518787162047535:2078] self: [2:7571518787162047534:0] peer: [3:7571518785770388836:0] socket: 349 qp: -1
2025-11-11T17:19:52.645002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518787162047598:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:52.645032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:52.665845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518787162047533:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.665943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:52.665997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518787162047533:2066] [node 5] ICP22 created new session: [2:7571518787162047600:2066]
2025-11-11T17:19:52.666043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518787162047600:2066] [node 5] ICS09 handshake done sender: [2:7571518787162047539:2080] self: [2:7571518787162047538:0] peer: [5:7571518787830320901:0] socket: 350 qp: -1
2025-11-11T17:19:52.666080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518787162047600:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:52.666117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:52.714651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518782867080206:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(18482570ns) {HandleHandshakeStatus:329(18479812ns) {DropIncomingHandshake:441(220ns) DropOutgoingHandshake:456(669ns) SwitchToState:178(208ns) ProcessPendingSessionEvents:495(18164339ns) {ForwardSessionEventToSession:612(58086ns) ForwardSessionEventToSession:612(6917ns) ForwardSessionEventToSession:612(3109ns) ForwardSessionEventToSession:612(5716ns) ForwardSessionEventToSession:612(3612ns) ForwardSessionEventToSession:612(2625ns) ForwardSessionEventToSession:612(5222ns) ForwardSessionEventToSession:612(2233ns) ForwardSessionEventToSession:612(798220ns) ForwardSessionEventToSession:612(17264481ns)}}}
2025-11-11T17:19:52.729968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518787162047592:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.746054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518786431837089:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54563
2025-11-11T17:19:52.590287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518781475421454:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(34435071ns) {HandleHandshakeStatus:329(34432820ns) {DropIncomingHandshake:441(1308ns) DropOutgoingHandshake:456(4309ns) SwitchToState:178(459ns) ProcessPendingSessionEvents:495(94204ns) {ForwardSessionEventToSession:612(45447ns) ForwardSessionEventToSession:612(4684ns) ForwardSessionEventToSession:612(5644ns) ForwardSessionEventToSession:612(3429ns) ForwardSessionEventToSession:612(6494ns) ForwardSessionEventToSession:612(2482ns) ForwardSessionEventToSession:612(8393ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(2276ns) ForwardSessionEventToSession:612(5182ns)}}}
2025-11-11T17:19:52.769872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518786431837025:2068] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.636079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518787078657692:2050] [node 3] ICS09 handshake done sender: [4:7571518787078657640:2083] self: [4:7571518787078657683:0] peer: [3:7571518781475421469:0] socket: 355 qp: -1
2025-11-11T17:19:52.706266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.769961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.770011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518786431837025:2068] [node 4] ICP22 created new session: [1:7571518786431837090:2068]
2025-11-11T17:19:52.749853Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518787162047578:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.737968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518787830320847:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.645822Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518785770388787:2079] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.770064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518786431837090:2068] [node 4] ICS09 handshake done sender: [1:7571518786431837061:2112] self: [1:7571518786431837060:0] peer: [4:7571518787078657698:0] socket: 399 qp: -1
2025-11-11T17:19:52.770107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518786431837090:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:52.770147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:52.770657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518786431837025:2068] [node 4] ICP35 event processing took too much time PendingConnection:278(19610439ns) {HandleHandshakeStatus:329(19608054ns) {DropIncomingHandshake:441(18605ns) DropOutgoingHandshake:456(359ns) SwitchToState:178(315ns) ProcessPendingSessionEvents:495(105361ns) {ForwardSessionEventToSession:612(58839ns) ForwardSessionEventToSession:612(3283ns) ForwardSessionEventToSession:612(5477ns) ForwardSessionEventToSession:612(5797ns) ForwardSessionEventToSession:612(2738ns) ForwardSessionEventToSession:612(5520ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(3309ns) ForwardSessionEventToSession:612(5073ns) ForwardSessionEventToSession:612(3160ns)}}}
2025-11-11T17:19:52.725999Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518787952094631:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.726144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518783657127245:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.733708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518783657127244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518787952094624:2117] held: no
2025-11-11T17:19:52.786422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094632:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.749893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518787162047578:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.636109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518787078657692:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:52.781764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518787830320847:2060] [node 4] ICP35 event processing took too much time PendingConnection:278(19652485ns) {IncomingHandshake:261(19650774ns) {DropIncomingHandshake:441(297ns) PrepareNewSessionHandshake:476(1399ns) {SwitchToState:178(504ns)} GenerateSessionVirtualId:988(326ns) IssueIncomingHandshakeReply:193(71696ns) {SwitchToState:178(370ns)}}}
2025-11-11T17:19:52.645856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518785770388787:2079] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.636169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:52.790858Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837089:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.750454Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:52.661718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518785770388846:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59661
2025-11-11T17:19:52.668241Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518785770388787:2079] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.797887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518782136869694:2048] [node 2] ICP18 handshake (actor [1:7571518786431837089:2119]) is too old
2025-11-11T17:19:52.810083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518783657127244:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(64276273ns) {HandleHandshakeStatus:395(64271449ns) {LogHandshakeFail:472(11193ns) {UpdateErrorStateLog:544(4315ns)} DropOutgoingHandshake:456(3278ns) ProcessPendingSessionEvents:495(44470351ns) {DropSessionEvent:512(20786ns) DropSessionEvent:512(5893ns) DropSessionEvent:512(7437ns) DropSessionEvent:512(5890ns) DropSessionEvent:512(5102ns) DropSessionEvent:512(4833ns) DropSessionEvent:512(5167ns) DropSessionEvent:512(14999252ns) DropSessionEvent:512(5503ns) DropSessionEvent:512(24530283ns)}}}
2025-11-11T17:19:52.782142Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9807) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.636562Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518787078657681:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.776144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518787162047603:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20262
2025-11-11T17:19:52.782945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518787830320885:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:52.674415Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518785770388788:2080] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:52.811900Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518787952094624:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:52.836423Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518786431837074:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.668294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518787078657634:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.785159Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518787830320886:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:52.789828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518783535353524:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9807) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518787830320885:2111] held: no
2025-11-11T17:19:52.790129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518783535353525:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.790191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:52.790231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518783535353525:2050] [node 3] ICP22 created new session: [5:7571518787830320909:2050]
2025-11-11T17:19:52.790280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518787830320909:2050] [node 3] ICS09 handshake done sender: [5:7571518787830320886:2112] self: [5:7571518787830320902:0] peer: [3:7571518785770388786:0] socket: 367 qp: -1
2025-11-11T17:19:52.790308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518787830320909:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:52.790359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:52.798739Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:52.801151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320907:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.801311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518787830320847:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518787830320904:2119] held: no
2025-11-11T17:19:52.801458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320908:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.804645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518787830320907:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.804730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518783535353524:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.804794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:52.804836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518783535353524:2049] [node 2] ICP22 created new session: [5:7571518787830320912:2049]
2025-11-11T17:19:52.811089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518787830320912:2049] [node 2] ICS09 handshake done sender: [5:7571518787830320885:2111] self: [5:7571518787830320901:0] peer: [2:7571518787162047538:0] socket: 366 qp: -1
2025-11-11T17:19:52.811141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518787830320912:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:52.811184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:52.818002Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518787830320904:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:52.822205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518787830320847:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.822322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:52.822370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518787830320847:2060] [node 4] ICP22 created new session: [5:7571518787830320914:2060]
2025-11-11T17:19:52.822413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518787830320914:2060] [node 4] ICS09 handshake done sender: [5:7571518787830320904:2119] self: [5:7571518787830320906:0] peer: [4:7571518787078657684:0] socket: 406 qp: -1
2025-11-11T17:19:52.822443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518787830320914:2060] [node 4] ICS10 traffic start
2025-11-11T17:19:52.822480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:52.822919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518783535353523:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.840939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518787830320917:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20740
2025-11-11T17:19:52.776515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518782867080206:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:52.783323Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518787162047588:2122] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:52.789731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518782867080205:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.797823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:52.802909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518782867080205:2048] [node 1] ICP22 created new session: [2:7571518787162047604:2048]
2025-11-11T17:19:52.818240Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518787162047578:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:52.833769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518787162047604:2048] [node 1] ICS09 handshake done sender: [2:7571518787162047588:2122] self: [2:7571518787162047596:0] peer: [1:7571518786431837056:0] socket: 383 qp: -1
2025-11-11T17:19:52.833826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518787162047604:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:52.833880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:52.846055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518787162047537:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.846113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:52.846168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518787162047537:2072] [node 6] ICP22 created new session: [2:7571518787162047607:2072]
2025-11-11T17:19:52.834622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518783657127244:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.668346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:52.689991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518787078657634:2066] [node 6] ICP22 created new session: [4:7571518787078657696:2066]
2025-11-11T17:19:52.700611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657697:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
2025-11-11T17:19:52.702136Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518787078657697:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.702231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518782783690305:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.704466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518787078657699:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57367
2025-11-11T17:19:52.715710Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518787078657697:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:52.715931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518782783690305:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.717654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:52.725736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518787078657696:2066] [node 6] ICS09 handshake done sender: [4:7571518787078657643:2085] self: [4:7571518787078657642:0] peer: [6:7571518787952094626:0] socket: 385 qp: -1
2025-11-11T17:19:52.730041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518782783690305:2048] [node 1] ICP22 created new session: [4:7571518787078657700:2048]
2025-11-11T17:19:52.733852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518787078657696:2066] [node 6] ICS10 traffic start
2025-11-11T17:19:52.733910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:52.754131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518787078657634:2066] [node 6] ICP35 event processing took too much time PendingConnection:278(108194775ns) {HandleHandshakeStatus:329(97233387ns) {DropIncomingHandshake:441(15087ns) DropOutgoingHandshake:456(726ns) SwitchToState:178(254ns) ProcessPendingSessionEvents:495(117212ns) {ForwardSessionEventToSession:612(67249ns) ForwardSessionEventToSession:612(4056ns) ForwardSessionEventToSession:612(3215ns) ForwardSessionEventToSession:612(6096ns) ForwardSessionEventToSession:612(4990ns) ForwardSessionEventToSession:612(2032ns) ForwardSessionEventToSession:612(6185ns) ForwardSessionEventToSession:612(2142ns) ForwardSessionEventToSession:612(10103ns) ForwardSessionEventToSession:612(2934ns)}}}
2025-11-11T17:19:52.754961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518787078657700:2048] [node 1] ICS09 handshake done sender: [4:7571518787078657697:2121] self: [4:7571518787078657698:0] peer: [1:7571518786431837060:0] socket: 401 qp: -1
2025-11-11T17:19:52.778095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518787078657700:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:52.784053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518787078657688:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.784085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518787078657688:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.785153Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518787078657688:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:52.785373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518787078657652:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.785419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:52.785459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518787078657652:2069] [node 5] ICP22 created new session: [4:7571518787078657703:2069]
2025-11-11T17:19:52.785954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:52.787774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518787078657703:2069] [node 5] ICS09 handshake done sender: [4:7571518787078657688:2119] self: [4:7571518787078657684:0] peer: [5:7571518787830320906:0] socket: 404 qp: -1
2025-11-11T17:19:52.787818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518787078657703:2069] [node 5] ICS10 traffic start
2025-11-11T17:19:52.787875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:52.869985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518782783690305:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(153804748ns) {HandleHandshakeStatus:329(153802670ns) {DropIncomingHandshake:441(1268ns) DropOutgoingHandshake:456(4641ns) SwitchToState:178(401ns) ProcessPendingSessionEvents:495(64430822ns) {ForwardSessionEventToSession:612(32585018ns) ForwardSessionEventToSession:612(7434ns) ForwardSessionEventToSession:612(8606236ns) ForwardSessionEventToSession:612(804072ns) ForwardSessionEventToSession:612(5742ns) ForwardSessionEventToSession:612(10361340ns) ForwardSessionEventToSession:612(4252ns) ForwardSessionEventToSession:612(3448ns) ForwardSessionEventToSession:612(12018743ns) ForwardSessionEventToSession:612(9635ns)}}}
2025-11-11T17:19:52.846213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518787162047607:2072] [node 6] ICS09 handshake done sender: [2:7571518787162047578:2114] self: [2:7571518787162047569:0] peer: [6:7571518787952094633:0] socket: 358 qp: -1
2025-11-11T17:19:52.675387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518781475421455:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.834694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:52.834739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518783657127244:2048] [node 1] ICP22 created new session: [6:7571518787952094636:2048]
2025-11-11T17:19:52.834782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518787952094636:2048] [node 1] ICS09 handshake done sender: [6:7571518787952094624:2117] self: [6:7571518787952094628:0] peer: [1:7571518786431837058:0] socket: 382 qp: -1
2025-11-11T17:19:52.834811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518787952094636:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:52.834840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:52.835363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094635:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.839472Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518787952094631:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:52.841147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518783657127245:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.841195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:52.841234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518783657127245:2049] [node 2] ICP22 created new session: [6:7571518787952094638:2049]
2025-11-11T17:19:52.841274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518787952094638:2049] [node 2] ICS09 handshake done sender: [6:7571518787952094631:2120] self: [6:7571518787952094633:0] peer: [2:7571518787162047569:0] socket: 400 qp: -1
2025-11-11T17:19:52.841301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518787952094638:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:52.841335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:52.854864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094634:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.836475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518786431837074:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:52.675429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:52.675474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518781475421455:2049] [node 2] ICP22 created new session: [3:7571518785770388847:2049]
2025-11-11T17:19:52.675513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518785770388847:2049] [node 2] ICS09 handshake done sender: [3:7571518785770388788:2080] self: [3:7571518785770388836:0] peer: [2:7571518787162047534:0] socket: 352 qp: -1
2025-11-11T17:19:52.675539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518785770388847:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:52.675569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:52.725823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518785770388782:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.734085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:52.734136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518785770388782:2065] [node 5] ICP22 created new session: [3:7571518785770388849:2065]
2025-11-11T17:19:52.734202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518785770388849:2065] [node 5] ICS09 handshake done sender: [3:7571518785770388787:2079] self: [3:7571518785770388786:0] peer: [5:7571518787830320902:0] socket: 353 qp: -1
2025-11-11T17:19:52.734238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518785770388849:2065] [node 5] ICS10 traffic start
2025-11-11T17:19:52.734268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:52.734635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518785770388782:2065] [node 5] ICP35 event processing took too much time PendingConnection:278(34047283ns) {HandleHandshakeStatus:329(27990593ns) {DropIncomingHandshake:441(1763ns) DropOutgoingHandshake:456(1406ns) SwitchToState:178(3344ns) ProcessPendingSessionEvents:495(93198ns) {ForwardSessionEventToSession:612(47836ns) ForwardSessionEventToSession:612(3578ns) ForwardSessionEventToSession:612(5464ns) ForwardSessionEventToSession:612(2022ns) ForwardSessionEventToSession:612(2735ns) ForwardSessionEventToSession:612(7576ns) ForwardSessionEventToSession:612(2633ns) ForwardSessionEventToSession:612(3410ns) ForwardSessionEventToSession:612(5820ns) ForwardSessionEventToSession:612(2549ns)}}}
2025-11-11T17:19:52.846242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518787162047607:2072] [node 6] ICS10 traffic start
2025-11-11T17:19:52.895393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518787952094634:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.895555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518783657127291:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:52.846261Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518786431837074:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:52.846277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:52.895932Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518787830320907:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:52.854048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518786431837014:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(164619880ns) {HandleHandshakeStatus:329(164617875ns) {DropIncomingHandshake:441(13155ns) DropOutgoingHandshake:456(260ns) SwitchToState:178(93ns) ProcessPendingSessionEvents:495(51799817ns) {ForwardSessionEventToSession:612(34885923ns) ForwardSessionEventToSession:612(1610170ns) ForwardSessionEventToSession:612(4730ns) ForwardSessionEventToSession:612(3213ns) ForwardSessionEventToSession:612(1057766ns) ForwardSessionEventToSession:612(2596ns) ForwardSessionEventToSession:612(4062ns) ForwardSessionEventToSession:612(2200909ns) ForwardSessionEventToSession:612(6858292ns) ForwardSessionEventToSession:612(5148979ns)}}}
2025-11-11T17:19:52.867332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518782867080205:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(77424265ns) {HandleHandshakeStatus:329(77421571ns) {DropIncomingHandshake:441(1166ns) DropOutgoingHandshake:456(10474ns) SwitchToState:178(376ns) ProcessPendingSessionEvents:495(20066718ns) {ForwardSessionEventToSession:612(20010761ns) ForwardSessionEventToSession:612(5567ns) ForwardSessionEventToSession:612(6056ns) ForwardSessionEventToSession:612(7760ns) ForwardSessionEventToSession:612(2592ns) ForwardSessionEventToSession:612(2818ns) ForwardSessionEventToSession:612(6280ns) ForwardSessionEventToSession:612(2487ns) ForwardSessionEventToSession:612(9915ns) ForwardSessionEventToSession:612(3428ns)}}}
2025-11-11T17:19:52.870856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518787162047537:2072] [node 6] ICP35 event processing took too much time PendingConnection:278(24581154ns) {HandleHandshakeStatus:329(24578839ns) {DropIncomingHandshake:441(13260ns) DropOutgoingHandshake:456(343ns) SwitchToState:178(559ns) ProcessPendingSessionEvents:495(23163194ns) {ForwardSessionEventToSession:612(23113845ns) ForwardSessionEventToSession:612(6001ns) ForwardSessionEventToSession:612(3373ns) ForwardSessionEventToSession:612(9062ns) ForwardSessionEventToSession:612(3274ns) ForwardSessionEventToSession:612(5064ns) ForwardSessionEventToSession:612(2398ns) ForwardSessionEventToSession:612(2891ns) ForwardSessionEventToSession:612(5996ns) ForwardSessionEventToSession:612(2537ns)}}}
2025-11-11T17:19:52.857761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518786431837012:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.857831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:52.857882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518786431837012:2064] [node 5] ICP22 created new session: [1:7571518786431837093:2064]
2025-11-11T17:19:52.857933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518786431837093:2064] [node 5] ICS09 handshake done sender: [1:7571518786431837074:2117] self: [1:7571518786431837073:0] peer: [5:7571518787830320916:0] socket: 395 qp: -1
2025-11-11T17:19:52.857969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518786431837093:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:52.857997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:52.896004Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:52.897908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518783535353523:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518787830320907:2121] held: no
2025-11-11T17:19:52.898209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518783535353523:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.898285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:52.898328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518783535353523:2048] [node 1] ICP22 created new session: [5:7571518787830320918:2048]
2025-11-11T17:19:52.898375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518787830320918:2048] [node 1] ICS09 handshake done sender: [5:7571518787830320907:2121] self: [5:7571518787830320916:0] peer: [1:7571518786431837073:0] socket: 365 qp: -1
2025-11-11T17:19:52.898408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518787830320918:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:52.898467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:52.906025Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518787830320898:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:52.906070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518787830320898:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:52.908408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518787952094641:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48974
2025-11-11T17:19:52.908769Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518787830320898:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:52.909065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518787830320888:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:52.909166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:52.909210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518787830320888:2065] [node 6] ICP22 created new session: [5:7571518787830320921:2065]
2025-11-11T17:19:52.909258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518787830320921:2065] [node 6] ICS09 handshake done sender: [5:7571518787830320898:2116] self: [5:7571518787830320897:0] peer: [6:7571518787952094640:0] socket: 364 qp: -1
2025-11-11T17:19:52.909289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518787830320921:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:52.909329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:52.910818Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518787952094634:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:52.911154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518783657127291:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:52.911208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:52.911244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518783657127291:2063] [node 5] ICP22 created new session: [6:7571518787952094642:2063]
2025-11-11T17:19:52.911293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518787952094642:2063] [node 5] ICS09 handshake done sender: [6:7571518787952094634:2122] self: [6:7571518787952094640:0] peer: [5:7571518787830320897:0] socket: 375 qp: -1
2025-11-11T17:19:52.911331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518787952094642:2063] [node 5] ICS10 traffic start
2025-11-11T17:19:52.911358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:52.969914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:52.970143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518787162047604:2048] [node 1] ICS07 socket disconnect 383 reason# EndOfStream
2025-11-11T17:19:52.970173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518787162047604:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:52.970250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:52.970293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518787162047604:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:52.978944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:52.979168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518785770388843:2048] [node 1] ICS01 socket: 384 reason# EndOfStream
2025-11-11T17:19:52.979207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518781475421454:2048] [node 1] ICP30 unregister session Session# [3:7571518785770388843:2048] VirtualId# [3:7571518785770388837:0]
2025-11-11T17:19:52.979241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518785770388843:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:52.979331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:52.979371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:52.980293Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:19:52.981479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518782867080205:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:19:52.981514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518787162047604:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:52.982143Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address
2025-11-11T17:19:52.982392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518782867080205:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:52.982430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518787162047604:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:52.982460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518782867080205:2048] [node 1] ICP30 unregister session Session# [2:7571518787162047604:2048] VirtualId# [2:7571518787162047596:0]
2025-11-11T17:19:52.982506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:52.982531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518782867080205:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:52.982546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:52.985768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:52.985994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518787952094636:2048] [node 1] ICS01 socket: 382 reason# EndOfStream
2025-11-11T17:19:52.986035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518783657127244:2048] [node 1] ICP30 unregister session Session# [6:7571518787952094636:2048] VirtualId# [6:7571518787952094628:0]
2025-11-11T17:19:52.986068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518787952094636:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:52.986142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:52.986180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:52.989788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:52.990010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518787830320918:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-11-11T17:19:52.990068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518783535353523:2048] [node 1] ICP30 unregister session Session# [5:7571518787830320918:2048] VirtualId# [5:7571518787830320916:0]
2025-11-11T17:19:52.990099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518787830320918:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:52.990184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:52.990229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:52.991756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:52.992030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518787078657700:2048] [node 1] ICS07 socket disconnect 401 reason# EndOfStream
2025-11-11T17:19:52.992070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518787078657700:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:52.992183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:52.992229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518787078657700:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:52.993316Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address
2025-11-11T17:19:52.993567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518782783690305:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:52.993624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518787078657700:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:52.993660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518782783690305:2048] [node 1] ICP30 unregister session Session# [4:7571518787078657700:2048] VirtualId# [4:7571518787078657698:0]
2025-11-11T17:19:52.993704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:52.993732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518782783690305:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:52.993746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:52.993665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518782867080205:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:53.009710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518782783690305:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:53.129809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:53.130096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518785770388847:2049] [node 2] ICS01 socket: 352 reason# EndOfStream
2025-11-11T17:19:53.130146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518781475421455:2049] [node 2] ICP30 unregister session Session# [3:7571518785770388847:2049] VirtualId# [3:7571518785770388836:0]
2025-11-11T17:19:53.130182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518785770388847:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.130288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:53.130330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:53.147737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:53.147969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518787078657685:2049] [node 2] ICS07 socket disconnect 356 reason# EndOfStream
2025-11-11T17:19:53.147995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518787078657685:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.148077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:53.148116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518787078657685:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:53.149111Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9807) Couldn't connect to any resolved address
2025-11-11T17:19:53.149362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518782783690306:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9807) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:53.149403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518787078657685:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:53.149435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518782783690306:2049] [node 2] ICP30 unregister session Session# [4:7571518787078657685:2049] VirtualId# [4:7571518787078657678:0]
2025-11-11T17:19:53.149478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:53.149506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518782783690306:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9807) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:53.149521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9807) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:53.149878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:53.150105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518787830320912:2049] [node 2] ICS01 socket: 366 reason# EndOfStream
2025-11-11T17:19:53.150146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518783535353524:2049] [node 2] ICP30 unregister session Session# [5:7571518787830320912:2049] VirtualId# [5:7571518787830320901:0]
2025-11-11T17:19:53.150174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518787830320912:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.150236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:53.150273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:53.153935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:53.154136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518787952094638:2049] [node 2] ICS01 socket: 400 reason# EndOfStream
2025-11-11T17:19:53.154166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518783657127245:2049] [node 2] ICP30 unregister session Session# [6:7571518787952094638:2049] VirtualId# [6:7571518787952094633:0]
2025-11-11T17:19:53.154193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518787952094638:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.154269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:53.154306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:53.159293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518782783690306:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:53.245729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:53.245941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518787078657692:2050] [node 3] ICS01 socket: 355 reason# EndOfStream
2025-11-11T17:19:53.245982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518782783690307:2050] [node 3] ICP30 unregister session Session# [4:7571518787078657692:2050] VirtualId# [4:7571518787078657683:0]
2025-11-11T17:19:53.246015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518787078657692:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.246086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:53.246128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:53.245705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:53.245941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518787952094621:2050] [node 3] ICS01 socket: 357 reason# EndOfStream
2025-11-11T17:19:53.245980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518783657127246:2050] [node 3] ICP30 unregister session Session# [6:7571518787952094621:2050] VirtualId# [6:7571518787952094619:0]
2025-11-11T17:19:53.246014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518787952094621:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.246086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:53.246124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:53.245730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:53.245939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518787830320909:2050] [node 3] ICS07 socket disconnect 367 reason# EndOfStream
2025-11-11T17:19:53.245968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518787830320909:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.246060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:53.246104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518787830320909:2050] [node 3] ICS15 start handshake
2025-11-11T17:19:53.253657Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3561) Couldn't connect to any resolved address
2025-11-11T17:19:53.261876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518783535353525:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3561) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:53.261944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518787830320909:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:53.261983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518783535353525:2050] [node 3] ICP30 unregister session Session# [5:7571518787830320909:2050] VirtualId# [5:7571518787830320902:0]
2025-11-11T17:19:53.262041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:53.262074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518783535353525:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3561) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:53.262091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3561) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:53.273685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518783535353525:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:19:53.393770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:53.393977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518787830320914:2060] [node 4] ICS01 socket: 406 reason# EndOfStream
2025-11-11T17:19:53.394029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518787830320847:2060] [node 4] ICP30 unregister session Session# [5:7571518787830320914:2060] VirtualId# [5:7571518787830320906:0]
2025-11-11T17:19:53.394074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518787830320914:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.394153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:53.394194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:53.397819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:53.398085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518787952094629:2061] [node 4] ICS01 socket: 386 reason# EndOfStream
2025-11-11T17:19:53.398119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518783657127261:2061] [node 4] ICP30 unregister session Session# [6:7571518787952094629:2061] VirtualId# [6:7571518787952094626:0]
2025-11-11T17:19:53.398179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518787952094629:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.398338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:53.398421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:53.460465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:53.461237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518787952094642:2063] [node 5] ICS01 socket: 375 reason# EndOfStream
2025-11-11T17:19:53.461281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518783657127291:2063] [node 5] ICP30 unregister session Session# [6:7571518787952094642:2063] VirtualId# [6:7571518787952094640:0]
2025-11-11T17:19:53.461316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518787952094642:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:53.461464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:53.461515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:53.649268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518792545457289:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:53.649375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518792545457288:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:53.706920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518789986960872:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:53.706966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518789986960873:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:53.749509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518792083920090:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:53.749557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518792083920091:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:53.823152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518790197902800:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:53.823208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518790197902801:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:53.823236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518790197902802:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:53.885895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518791020624073:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:53.885942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518791020624074:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:53.885969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518791020624075:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:53.925963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518791160371764:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:53.926003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518791160371765:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:53.926027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518791160371766:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:54.116751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518796378887406:2064] [node 4] ICP01 ready to work
2025-11-11T17:19:54.109265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518794281928181:2060] [node 4] ICP01 ready to work
2025-11-11T17:19:54.114201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518795315591388:2066] [node 4] ICP01 ready to work
2025-11-11T17:19:54.118211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518796840424616:2079] [node 4] ICP01 ready to work
2025-11-11T17:19:54.197973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518796378887411:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:54.126620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518794281928184:2063] [node 6] ICP01 ready to work
2025-11-11T17:19:54.209685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518796840424628:2076] [node 5] ICP01 ready to work
2025-11-11T17:19:54.209822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518796378887410:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.360209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870120:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.360507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870138:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
2025-11-11T17:19:54.362639Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518794492870138:2088] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.362707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518794492870120:2070] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.290368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518794281928184:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(163439858ns) {Configure:104(163435787ns) {ProcessConfigured:135(44308428ns) {StartInitialHandshake:157(44304568ns) {DropHandshakes:469(2851ns) {DropIncomingHandshake:441(704ns) DropOutgoingHandshake:456(645ns)} GenerateSessionVirtualId:988(816ns) PrepareNewSessionHandshake:476(5538ns) {SwitchToState:178(2006ns)}}}}}
2025-11-11T17:19:54.372294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518794492870150:2072] [node 5] ICP01 ready to work
2025-11-11T17:19:54.372347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518796840424644:2086] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.384279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518796378887458:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33071
2025-11-11T17:19:54.389428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518794281928237:2066] [node 5] ICP01 ready to work
2025-11-11T17:19:54.395081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518795455339116:2073] [node 5] ICP01 ready to work
2025-11-11T17:19:54.417927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518795455339117:2069] [node 4] ICP01 ready to work
2025-11-11T17:19:54.377320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518795315591417:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:54.374516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518790197902800:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.410435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518792545457288:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(316523810ns) {Configure:104(316518344ns) {ProcessConfigured:135(58995ns) {StartInitialHandshake:157(56469ns) {DropHandshakes:469(1881ns) {DropIncomingHandshake:441(370ns) DropOutgoingHandshake:456(247ns)} GenerateSessionVirtualId:988(899ns) PrepareNewSessionHandshake:476(1575ns) {SwitchToState:178(843ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.458914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518795455339116:2073] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16762027ns) {Configure:104(16757711ns) {ProcessConfigured:135(66923ns) {StartInitialHandshake:157(64188ns) {DropHandshakes:469(4519ns) {DropIncomingHandshake:441(521ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:476(1881ns) {SwitchToState:178(626ns)}}}}}
2025-11-11T17:19:54.461080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339124:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.461241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339125:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.461286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339126:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.461498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339130:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.391759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518794281928228:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7768
2025-11-11T17:19:54.391908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518794281928234:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7768
2025-11-11T17:19:54.392021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518794281928235:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7768
2025-11-11T17:19:54.448251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928228:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.463023Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928234:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.470267Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518795455339126:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.470350Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518795455339124:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.470612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518795455339116:2073] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.446534Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518796378887458:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.437821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518794492870169:2094] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.494550Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518795315591440:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.494586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518795315591440:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.495441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591444:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.425373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518796840424656:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36456
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.486721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518796378887410:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(192318446ns) {Configure:104(192312007ns) {ProcessConfigured:135(74988ns) {StartInitialHandshake:157(73270ns) {DropHandshakes:469(2709ns) {DropIncomingHandshake:441(645ns) DropOutgoingHandshake:456(502ns)} GenerateSessionVirtualId:988(674ns) PrepareNewSessionHandshake:476(1573ns) {SwitchToState:178(435ns)}}}}}
2025-11-11T17:19:54.461817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870176:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.425432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518796840424659:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36456
2025-11-11T17:19:54.496033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518794281928181:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(386468148ns) {Configure:104(386464021ns) {ProcessConfigured:135(23215937ns) {StartInitialHandshake:157(23213226ns) {DropHandshakes:469(2183ns) {DropIncomingHandshake:441(367ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:476(3354ns) {SwitchToState:178(1223ns)}}}}}
2025-11-11T17:19:54.516541Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518795455339130:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.479982Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518794492870138:2088] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:54.509725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518796378887406:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(324395926ns) {Configure:104(324392240ns) {ProcessConfigured:135(15141490ns) {StartInitialHandshake:157(15139637ns) {DropHandshakes:469(1405ns) {DropIncomingHandshake:441(339ns) DropOutgoingHandshake:456(299ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:476(2930ns) {SwitchToState:178(130ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.521947Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518795315591440:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:54.427432Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424618:2080] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.496226Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928235:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.586001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518792083920090:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(364074470ns) {Configure:104(364069633ns) {ProcessConfigured:135(7954831ns) {StartInitialHandshake:157(7952364ns) {DropHandshakes:469(2432ns) {DropIncomingHandshake:441(640ns) DropOutgoingHandshake:456(384ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:476(3360ns) {SwitchToState:178(1206ns)}}}}}
2025-11-11T17:19:54.586252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518792083920090:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.517054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339136:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.602690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518796378887467:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33071
2025-11-11T17:19:54.503811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518794281928245:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7768
2025-11-11T17:19:54.480119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870177:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
2025-11-11T17:19:54.574511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518791020624073:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(404581711ns) {Configure:104(404577134ns) {ProcessConfigured:135(55511ns) {StartInitialHandshake:157(52892ns) {DropHandshakes:469(2349ns) {DropIncomingHandshake:441(510ns) DropOutgoingHandshake:456(482ns)} GenerateSessionVirtualId:988(482ns) PrepareNewSessionHandshake:476(1421ns) {SwitchToState:178(568ns)}}}}}
2025-11-11T17:19:54.427470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518796840424618:2080] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.480183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870178:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
2025-11-11T17:19:54.517133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339137:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.589836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591443:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.443296Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518796840424618:2080] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.525870Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928245:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.443577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518796840424616:2079] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.443634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:54.443681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518796840424616:2079] [node 4] ICP22 created new session: [1:7571518796840424664:2079]
2025-11-11T17:19:54.443738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518796840424664:2079] [node 4] ICS09 handshake done sender: [1:7571518796840424618:2080] self: [1:7571518796840424617:0] peer: [4:7571518794492870167:0] socket: 355 qp: -1
2025-11-11T17:19:54.443771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518796840424664:2079] [node 4] ICS10 traffic start
2025-11-11T17:19:54.443805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:54.537790Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424658:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.537818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518796840424658:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.538536Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:54.538807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518796840424616:2079] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:54.553760Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518796840424658:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:54.586572Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424659:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.586615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518792545457288:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.586671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:54.586718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518792545457288:2048] [node 2] ICP22 created new session: [1:7571518796840424668:2048]
2025-11-11T17:19:54.586762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518796840424668:2048] [node 2] ICS09 handshake done sender: [1:7571518796840424658:2113] self: [1:7571518796840424657:0] peer: [2:7571518794281928248:0] socket: 373 qp: -1
2025-11-11T17:19:54.586795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518796840424668:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:54.586825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:54.587170Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424635:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.587191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518796840424635:2095] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.588029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518796840424635:2095] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:54.588219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518792545457289:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.588254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:54.588290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518792545457289:2049] [node 3] ICP22 created new session: [1:7571518796840424671:2049]
2025-11-11T17:19:54.588326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518796840424671:2049] [node 3] ICS09 handshake done sender: [1:7571518796840424635:2095] self: [1:7571518796840424634:0] peer: [3:7571518796378887466:0] socket: 362 qp: -1
2025-11-11T17:19:54.610054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518795315591444:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.588352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518796840424671:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:54.588376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:54.601869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518792545457288:2048] [node 2] ICP18 handshake (actor [1:7571518796840424659:2114]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.480349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518790197902800:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.550098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518795455339125:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.630069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518796378887468:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33071
2025-11-11T17:19:54.527628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518789986960872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.480393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:54.662636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518796840424673:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36456
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.641662Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518796378887458:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:54.560699Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518795455339137:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.610104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518795315591417:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.577264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518789986960873:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(466288665ns) {Configure:104(443404185ns) {ProcessConfigured:135(23386297ns) {StartInitialHandshake:157(15447497ns) {DropHandshakes:469(2649ns) {DropIncomingHandshake:441(600ns) DropOutgoingHandshake:456(323ns)} GenerateSessionVirtualId:988(1701ns) PrepareNewSessionHandshake:476(3925ns) {SwitchToState:178(1679ns)}}}}}
2025-11-11T17:19:54.593650Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518795455339126:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:54.594229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518791160371765:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(393107418ns) {Configure:104(393104350ns) {ProcessConfigured:135(9023536ns) {StartInitialHandshake:157(9021639ns) {DropHandshakes:469(725ns) {DropIncomingHandshake:441(239ns) DropOutgoingHandshake:456(213ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:476(2635ns) {SwitchToState:178(159ns)}}}}}
2025-11-11T17:19:54.594671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518791160371765:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.594913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518795455339117:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.598064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518791160371766:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(385030344ns) {Configure:104(385025798ns) {ProcessConfigured:135(47837ns) {StartInitialHandshake:157(46459ns) {DropHandshakes:469(1669ns) {DropIncomingHandshake:441(383ns) DropOutgoingHandshake:456(316ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:476(1148ns) {SwitchToState:178(382ns)}}}}}
2025-11-11T17:19:54.599826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518795455339116:2073] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.599894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:54.599936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518795455339116:2073] [node 5] ICP22 created new session: [6:7571518795455339145:2073]
2025-11-11T17:19:54.599983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518795455339145:2073] [node 5] ICS09 handshake done sender: [6:7571518795455339126:2112] self: [6:7571518795455339135:0] peer: [5:7571518795315591439:0] socket: 378 qp: -1
2025-11-11T17:19:54.600014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518795455339145:2073] [node 5] ICS10 traffic start
2025-11-11T17:19:54.600051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:54.626114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518791160371766:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.649887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.671424Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518795455339125:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:54.745011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518791160371764:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(282359011ns) {Configure:104(282355313ns) {ProcessConfigured:135(27408937ns) {StartInitialHandshake:157(27404936ns) {DropHandshakes:469(1924ns) {DropIncomingHandshake:441(431ns) DropOutgoingHandshake:456(439ns)} GenerateSessionVirtualId:988(1539ns) PrepareNewSessionHandshake:476(2413ns) {SwitchToState:178(843ns)}}}}}
2025-11-11T17:19:54.745066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518791160371765:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.745136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:54.745183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518791160371765:2049] [node 2] ICP22 created new session: [6:7571518795455339153:2049]
2025-11-11T17:19:54.745225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518795455339153:2049] [node 2] ICS09 handshake done sender: [6:7571518795455339125:2111] self: [6:7571518795455339140:0] peer: [2:7571518794281928206:0] socket: 377 qp: -1
2025-11-11T17:19:54.745261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518795455339153:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:54.745292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:54.747028Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:54.748340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518791160371764:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.480436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518790197902800:2048] [node 1] ICP22 created new session: [4:7571518794492870179:2048]
2025-11-11T17:19:54.610139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:54.665064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518792083920090:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.725914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518796840424628:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(405201866ns) {Configure:104(405194307ns) {ProcessConfigured:135(27994588ns) {StartInitialHandshake:157(27992154ns) {DropHandshakes:469(2412ns) {DropIncomingHandshake:441(570ns) DropOutgoingHandshake:456(357ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:476(5961ns) {SwitchToState:178(3503ns)}}}}}
2025-11-11T17:19:54.593819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928244:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.754028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518795455339117:2069] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518795455339137:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.480493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518794492870179:2048] [node 1] ICS09 handshake done sender: [4:7571518794492870138:2088] self: [4:7571518794492870167:0] peer: [1:7571518796840424617:0] socket: 360 qp: -1
2025-11-11T17:19:54.665993Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518796378887413:2068] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.726094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424673:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.593859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518794281928244:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.610180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518795315591417:2065] [node 6] ICP22 created new session: [5:7571518795315591448:2065]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.480528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518794492870179:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:54.610239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518795315591448:2065] [node 6] ICS09 handshake done sender: [5:7571518795315591440:2112] self: [5:7571518795315591439:0] peer: [6:7571518795455339135:0] socket: 371 qp: -1
2025-11-11T17:19:54.666023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518796378887413:2068] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.754221Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12161) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:54.603804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928224:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.769980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518791160371766:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12161) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518795455339124:2110] held: no
2025-11-11T17:19:54.805700Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424661:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.480561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.610276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518795315591448:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:54.669687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518796378887468:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.669763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:54.672286Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518796378887460:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.672306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518796378887460:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.673732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518796378887470:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33071
2025-11-11T17:19:54.674733Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518796378887413:2068] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:54.674987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518796378887460:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:54.675147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518796378887411:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.675196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:54.675242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518796378887411:2065] [node 6] ICP22 created new session: [3:7571518796378887474:2065]
2025-11-11T17:19:54.675279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518796378887474:2065] [node 6] ICS09 handshake done sender: [3:7571518796378887413:2068] self: [3:7571518796378887412:0] peer: [6:7571518795455339148:0] socket: 361 qp: -1
2025-11-11T17:19:54.675305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518796378887474:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:54.675337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:54.681719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518792083920090:2048] [node 1] ICP22 created new session: [3:7571518796378887469:2048]
2025-11-11T17:19:54.681767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518796378887469:2048] [node 1] ICS09 handshake done sender: [3:7571518796378887458:2112] self: [3:7571518796378887466:0] peer: [1:7571518796840424634:0] socket: 366 qp: -1
2025-11-11T17:19:54.681805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518796378887469:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:54.681851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:54.682279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518792083920090:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17011456ns) {HandleHandshakeStatus:329(17009624ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(4806ns) SwitchToState:178(577ns) ProcessPendingSessionEvents:495(93534ns) {ForwardSessionEventToSession:612(48758ns) ForwardSessionEventToSession:612(2995ns) ForwardSessionEventToSession:612(2378ns) ForwardSessionEventToSession:612(5343ns) ForwardSessionEventToSession:612(3040ns) ForwardSessionEventToSession:612(4933ns) ForwardSessionEventToSession:612(7995ns) ForwardSessionEventToSession:612(3067ns) ForwardSessionEventToSession:612(4949ns) ForwardSessionEventToSession:612(2753ns)}}}
2025-11-11T17:19:54.682346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518796378887406:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.682381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:54.682414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518796378887406:2064] [node 4] ICP22 created new session: [3:7571518796378887477:2064]
2025-11-11T17:19:54.682451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518796378887477:2064] [node 4] ICS09 handshake done sender: [3:7571518796378887460:2113] self: [3:7571518796378887459:0] peer: [4:7571518794492870189:0] socket: 384 qp: -1
2025-11-11T17:19:54.682477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518796378887477:2064] [node 4] ICS10 traffic start
2025-11-11T17:19:54.682503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:54.749924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518796378887480:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33071
2025-11-11T17:19:54.758978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518796378887470:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.802110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518792083920091:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.817213Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:19:54.817653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518796378887406:2064] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:54.833617Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518796378887463:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.833660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518796378887463:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.844713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518792083920091:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(38884399ns) {IncomingHandshake:261(16949684ns) {DropIncomingHandshake:441(967ns) PrepareNewSessionHandshake:476(1317ns) {SwitchToState:178(399ns)} GenerateSessionVirtualId:988(894ns) IssueIncomingHandshakeReply:193(7567993ns) {SwitchToState:178(4225ns)}}}
2025-11-11T17:19:54.853760Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518796378887463:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.603829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518794281928224:2101] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.805743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518796840424661:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.833832Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518796840424661:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:54.841639Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424675:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.841682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518796840424675:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.843627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518796840424644:2086] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.843670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:54.857990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518796840424679:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36456
2025-11-11T17:19:54.862617Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518796840424675:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:54.862865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518796840424628:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.862919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:54.862966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518796840424628:2076] [node 5] ICP22 created new session: [1:7571518796840424682:2076]
2025-11-11T17:19:54.863006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518796840424682:2076] [node 5] ICS09 handshake done sender: [1:7571518796840424675:2117] self: [1:7571518796840424674:0] peer: [5:7571518795315591460:0] socket: 388 qp: -1
2025-11-11T17:19:54.863040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518796840424682:2076] [node 5] ICS10 traffic start
2025-11-11T17:19:54.863076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:54.877682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518796840424644:2086] [node 6] ICP22 created new session: [1:7571518796840424680:2086]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.528215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870182:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
2025-11-11T17:19:54.562084Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518794492870182:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.562229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518790197902801:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.574882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518790197902802:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(392735058ns) {Configure:104(392730278ns) {ProcessConfigured:135(14341846ns) {StartInitialHandshake:157(14339806ns) {DropHandshakes:469(974417ns) {DropIncomingHandshake:441(661ns) DropOutgoingHandshake:456(793ns)} GenerateSessionVirtualId:988(2253ns) PrepareNewSessionHandshake:476(2398ns) {SwitchToState:178(189ns)}}}}}
2025-11-11T17:19:54.583179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518794492870178:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.625853Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518794492870177:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.629879Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518794492870174:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.629916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518794492870174:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.643982Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:54.650230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518794492870169:2094] [node 6] ICP35 event processing took too much time PendingConnection:278(31717690ns) {IncomingHandshake:261(12035522ns) {DropIncomingHandshake:441(493ns) PrepareNewSessionHandshake:476(2203ns) {SwitchToState:178(823ns)} GenerateSessionVirtualId:988(511ns) IssueIncomingHandshakeReply:193(11469ns)}}
2025-11-11T17:19:54.662095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518790197902802:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.665893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518790197902801:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518794492870182:2119] held: no
2025-11-11T17:19:54.666939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870187:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
2025-11-11T17:19:54.671775Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518794492870174:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:54.706082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518790197902800:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(153476337ns) {HandleHandshakeStatus:329(153474485ns) {DropIncomingHandshake:441(687ns) DropOutgoingHandshake:456(3856ns) SwitchToState:178(399ns) ProcessPendingSessionEvents:495(92104976ns) {ForwardSessionEventToSession:612(31966480ns) ForwardSessionEventToSession:612(7992360ns) ForwardSessionEventToSession:612(8132630ns) ForwardSessionEventToSession:612(5715682ns) ForwardSessionEventToSession:612(6087ns) ForwardSessionEventToSession:612(19588ns) ForwardSessionEventToSession:612(19848412ns) ForwardSessionEventToSession:612(4286ns) ForwardSessionEventToSession:612(3231ns) ForwardSessionEventToSession:612(7998974ns)}}}
2025-11-11T17:19:54.710272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518794492870169:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.710350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:54.710398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518794492870169:2094] [node 6] ICP22 created new session: [4:7571518794492870191:2094]
2025-11-11T17:19:54.710446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518794492870191:2094] [node 6] ICS09 handshake done sender: [4:7571518794492870174:2115] self: [4:7571518794492870173:0] peer: [6:7571518795455339141:0] socket: 390 qp: -1
2025-11-11T17:19:54.710478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518794492870191:2094] [node 6] ICS10 traffic start
2025-11-11T17:19:54.710532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:54.743520Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518794492870171:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.743544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518794492870171:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.750237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518794492870193:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35086
2025-11-11T17:19:54.753902Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518794492870171:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:54.784568Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518794492870177:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:54.789861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518794492870150:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.789922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:54.789964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518794492870150:2072] [node 5] ICP22 created new session: [4:7571518794492870195:2072]
2025-11-11T17:19:54.790356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518790197902802:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.790419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:54.790465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518790197902802:2050] [node 3] ICP22 created new session: [4:7571518794492870196:2050]
2025-11-11T17:19:54.790511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518794492870196:2050] [node 3] ICS09 handshake done sender: [4:7571518794492870177:2117] self: [4:7571518794492870189:0] peer: [3:7571518796378887459:0] socket: 387 qp: -1
2025-11-11T17:19:54.790545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518794492870196:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:54.790579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:54.805730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518790197902801:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(55105099ns) {HandleHandshakeStatus:395(55101662ns) {LogHandshakeFail:472(13064ns) {UpdateErrorStateLog:544(5841ns)} DropOutgoingHandshake:456(2857ns) ProcessPendingSessionEvents:495(36019001ns) {DropSessionEvent:512(18095ns) DropSessionEvent:512(5582ns) DropSessionEvent:512(6727ns) DropSessionEvent:512(31912935ns) DropSessionEvent:512(17592ns) DropSessionEvent:512(5769ns) DropSessionEvent:512(6442ns) DropSessionEvent:512(4024814ns) DropSessionEvent:512(5711ns) DropSessionEvent:512(4413ns)}}}
2025-11-11T17:19:54.809789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518794492870195:2072] [node 5] ICS09 handshake done sender: [4:7571518794492870171:2114] self: [4:7571518794492870170:0] peer: [5:7571518795315591453:0] socket: 375 qp: -1
2025-11-11T17:19:54.814963Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518794492870182:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:54.816236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518790197902801:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.816316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:54.816362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518790197902801:2049] [node 2] ICP22 created new session: [4:7571518794492870198:2049]
2025-11-11T17:19:54.816410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518794492870198:2049] [node 2] ICS09 handshake done sender: [4:7571518794492870182:2119] self: [4:7571518794492870184:0] peer: [2:7571518794281928243:0] socket: 367 qp: -1
2025-11-11T17:19:54.816441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518794492870198:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:54.816489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:54.826189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518794492870195:2072] [node 5] ICS10 traffic start
2025-11-11T17:19:54.833690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:54.882136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518794492870150:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(92052236ns) {HandleHandshakeStatus:329(92049477ns) {DropIncomingHandshake:441(15251ns) DropOutgoingHandshake:456(394ns) SwitchToState:178(343ns) ProcessPendingSessionEvents:495(16128964ns) {ForwardSessionEventToSession:612(16071403ns) ForwardSessionEventToSession:612(11512ns) ForwardSessionEventToSession:612(5461ns) ForwardSessionEventToSession:612(3208ns) ForwardSessionEventToSession:612(4904ns) ForwardSessionEventToSession:612(2325ns) ForwardSessionEventToSession:612(5496ns) ForwardSessionEventToSession:612(3585ns) ForwardSessionEventToSession:612(2498ns) ForwardSessionEventToSession:612(8426ns)}}}
2025-11-11T17:19:54.874453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518796378887483:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33071
2025-11-11T17:19:54.770388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339155:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.610750Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518794281928244:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:54.611090Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518794281928224:2101] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:54.611321Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:54.612439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518794281928181:2060] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.612485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:54.612526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518794281928181:2060] [node 4] ICP22 created new session: [2:7571518794281928254:2060]
2025-11-11T17:19:54.612564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518794281928254:2060] [node 4] ICS09 handshake done sender: [2:7571518794281928244:2115] self: [2:7571518794281928243:0] peer: [4:7571518794492870184:0] socket: 393 qp: -1
2025-11-11T17:19:54.612597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518794281928254:2060] [node 4] ICS10 traffic start
2025-11-11T17:19:54.612625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:54.612995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518794281928184:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.613046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:54.613086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518794281928184:2063] [node 6] ICP22 created new session: [2:7571518794281928256:2063]
2025-11-11T17:19:54.613119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518794281928256:2063] [node 6] ICS09 handshake done sender: [2:7571518794281928224:2101] self: [2:7571518794281928206:0] peer: [6:7571518795455339140:0] socket: 369 qp: -1
2025-11-11T17:19:54.613149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518794281928256:2063] [node 6] ICS10 traffic start
2025-11-11T17:19:54.613169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:54.613470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518789986960872:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518794281928245:2116] held: no
2025-11-11T17:19:54.701820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518794281928258:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7768
2025-11-11T17:19:54.702101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518794281928259:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7768
2025-11-11T17:19:54.703072Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518794281928245:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:54.704908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518789986960872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.704986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:54.705035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518789986960872:2048] [node 1] ICP22 created new session: [2:7571518794281928264:2048]
2025-11-11T17:19:54.705076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518794281928264:2048] [node 1] ICS09 handshake done sender: [2:7571518794281928245:2116] self: [2:7571518794281928248:0] peer: [1:7571518796840424657:0] socket: 391 qp: -1
2025-11-11T17:19:54.705109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518794281928264:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:54.705144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:54.717625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518794281928237:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(221140428ns) {Configure:104(221135193ns) {ProcessConfigured:135(47923858ns) {StartInitialHandshake:157(47921018ns) {DropHandshakes:469(1996ns) {DropIncomingHandshake:441(533ns) DropOutgoingHandshake:456(267ns)} GenerateSessionVirtualId:988(651ns) PrepareNewSessionHandshake:476(22110944ns) {SwitchToState:178(4331ns)}}}}}
2025-11-11T17:19:54.750453Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928259:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.762329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518794281928184:2063] [node 6] ICP18 handshake (actor [2:7571518794281928259:2119]) is too old
2025-11-11T17:19:54.813761Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928250:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.813786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518794281928250:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.817805Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518794281928262:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.817837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518794281928262:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:54.829746Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518794281928250:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:54.838847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518789986960873:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.838924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:54.838984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518789986960873:2049] [node 3] ICP22 created new session: [2:7571518794281928268:2049]
2025-11-11T17:19:54.839048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518794281928268:2049] [node 3] ICS09 handshake done sender: [2:7571518794281928250:2117] self: [2:7571518794281928249:0] peer: [3:7571518796378887481:0] socket: 363 qp: -1
2025-11-11T17:19:54.839081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518794281928268:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:54.839118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:54.610310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:54.658139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518791020624075:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(464139225ns) {Configure:104(464134413ns) {ProcessConfigured:135(12632298ns) {StartInitialHandshake:157(12629928ns) {DropHandshakes:469(2157ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(369ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:476(3757ns) {SwitchToState:178(1245ns)}}}}}
2025-11-11T17:19:54.665790Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518795315591443:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.709854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518795315591388:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.757667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518795315591417:2065] [node 6] ICP35 event processing took too much time PendingConnection:278(31600835ns) {HandleHandshakeStatus:329(31598526ns) {DropIncomingHandshake:441(278ns) DropOutgoingHandshake:456(583ns) SwitchToState:178(169ns) ProcessPendingSessionEvents:495(31262566ns) {ForwardSessionEventToSession:612(40782ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(4946ns) ForwardSessionEventToSession:612(4325ns) ForwardSessionEventToSession:612(4620ns) ForwardSessionEventToSession:612(2659ns) ForwardSessionEventToSession:612(2867ns) ForwardSessionEventToSession:612(15203246ns) ForwardSessionEventToSession:612(15973014ns) ForwardSessionEventToSession:612(6740ns)}}}
2025-11-11T17:19:54.758106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518795315591417:2065] [node 6] ICP18 handshake (actor [5:7571518795315591444:2114]) is too old
2025-11-11T17:19:54.782286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591455:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.782431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591456:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.782478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591457:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.789872Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518795315591455:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.790099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518791020624074:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.797892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591454:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.805978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518795315591454:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.806085Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518795315591456:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:54.806218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518791020624075:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.806266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518791020624073:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:54.806477Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518795315591443:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:54.816886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518795315591388:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.816931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:54.823644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518795315591388:2066] [node 4] ICP22 created new session: [5:7571518795315591461:2066]
2025-11-11T17:19:54.823704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518795315591461:2066] [node 4] ICS09 handshake done sender: [5:7571518795315591443:2113] self: [5:7571518795315591453:0] peer: [4:7571518794492870170:0] socket: 376 qp: -1
2025-11-11T17:19:54.823744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518795315591461:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:54.823795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:54.826231Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:54.830680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518795315591388:2066] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:54.874210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591463:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.878607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591464:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.883007Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:54.904384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518791020624073:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518795315591456:2119] held: no
2025-11-11T17:19:54.904573Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518795315591456:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:54.904791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518791020624073:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.904857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:54.904906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518791020624073:2048] [node 1] ICP22 created new session: [5:7571518795315591465:2048]
2025-11-11T17:19:54.904949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518795315591465:2048] [node 1] ICS09 handshake done sender: [5:7571518795315591456:2119] self: [5:7571518795315591460:0] peer: [1:7571518796840424674:0] socket: 370 qp: -1
2025-11-11T17:19:54.904983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518795315591465:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:54.905011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:54.905979Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518795315591454:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:54.889692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518796840424680:2086] [node 6] ICS09 handshake done sender: [1:7571518796840424661:2115] self: [1:7571518796840424660:0] peer: [6:7571518795455339156:0] socket: 380 qp: -1
2025-11-11T17:19:54.805940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339151:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.899160Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.894108Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518796840424679:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:54.897708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518796840424680:2086] [node 6] ICS10 traffic start
2025-11-11T17:19:54.917642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:54.930301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518796840424644:2086] [node 6] ICP35 event processing took too much time PendingConnection:278(86481155ns) {HandleHandshakeStatus:329(86476941ns) {DropIncomingHandshake:441(503ns) DropOutgoingHandshake:456(393ns) SwitchToState:178(348ns) ProcessPendingSessionEvents:495(102996ns) {ForwardSessionEventToSession:612(54657ns) ForwardSessionEventToSession:612(3735ns) ForwardSessionEventToSession:612(5480ns) ForwardSessionEventToSession:612(2371ns) ForwardSessionEventToSession:612(8366ns) ForwardSessionEventToSession:612(2676ns) ForwardSessionEventToSession:612(6338ns) ForwardSessionEventToSession:612(3930ns) ForwardSessionEventToSession:612(2837ns) ForwardSessionEventToSession:612(1663ns)}}}
2025-11-11T17:19:54.930591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518796840424644:2086] [node 6] ICP18 handshake (actor [1:7571518796840424679:2118]) is too old
2025-11-11T17:19:54.913859Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518796378887470:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:54.917940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518796378887410:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.917978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:54.922239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518792083920091:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518796378887470:2118] held: no
2025-11-11T17:19:54.922339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518792083920091:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.922417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:54.922457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518792083920091:2049] [node 2] ICP22 created new session: [3:7571518796378887485:2049]
2025-11-11T17:19:54.922511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518796378887485:2049] [node 2] ICS09 handshake done sender: [3:7571518796378887470:2118] self: [3:7571518796378887481:0] peer: [2:7571518794281928249:0] socket: 405 qp: -1
2025-11-11T17:19:54.922543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518796378887485:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:54.922589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:54.923527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518796378887487:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33071
2025-11-11T17:19:54.929711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518796378887410:2066] [node 5] ICP22 created new session: [3:7571518796378887484:2066]
2025-11-11T17:19:54.924499Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518794281928262:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:54.806740Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518795455339124:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:54.815415Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518795455339137:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:54.817956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518791160371766:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.826002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518795455339158:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41329
2025-11-11T17:19:54.829875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:54.829927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518791160371766:2050] [node 3] ICP22 created new session: [6:7571518795455339159:2050]
2025-11-11T17:19:54.829976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518795455339159:2050] [node 3] ICS09 handshake done sender: [6:7571518795455339124:2110] self: [6:7571518795455339148:0] peer: [3:7571518796378887412:0] socket: 374 qp: -1
2025-11-11T17:19:54.830005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518795455339159:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:54.830055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:54.837809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518795455339117:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.865736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:54.882968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518795455339117:2069] [node 4] ICP22 created new session: [6:7571518795455339161:2069]
2025-11-11T17:19:54.897795Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518795455339130:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:54.898985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518795455339161:2069] [node 4] ICS09 handshake done sender: [6:7571518795455339137:2117] self: [6:7571518795455339141:0] peer: [4:7571518794492870173:0] socket: 394 qp: -1
2025-11-11T17:19:54.906450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518795455339161:2069] [node 4] ICS10 traffic start
2025-11-11T17:19:54.911070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518791160371764:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.911137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:54.911184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518791160371764:2048] [node 1] ICP22 created new session: [6:7571518795455339162:2048]
2025-11-11T17:19:54.911240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518795455339162:2048] [node 1] ICS09 handshake done sender: [6:7571518795455339130:2114] self: [6:7571518795455339156:0] peer: [1:7571518796840424660:0] socket: 382 qp: -1
2025-11-11T17:19:54.911285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518795455339162:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:54.911316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:54.946384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:54.963118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518795455339117:2069] [node 4] ICP35 event processing took too much time PendingConnection:278(125222253ns) {HandleHandshakeStatus:329(125218958ns) {DropIncomingHandshake:441(665ns) DropOutgoingHandshake:456(346ns) SwitchToState:178(636ns) ProcessPendingSessionEvents:495(773ns)}}
2025-11-11T17:19:54.923741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518791020624075:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.923790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:54.973738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518794281928237:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:54.923831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518791020624075:2050] [node 3] ICP22 created new session: [5:7571518795315591468:2050]
2025-11-11T17:19:54.923868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518795315591468:2050] [node 3] ICS09 handshake done sender: [5:7571518795315591454:2117] self: [5:7571518795315591459:0] peer: [3:7571518796378887462:0] socket: 410 qp: -1
2025-11-11T17:19:54.923896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518795315591468:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:54.923923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:54.962004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518795315591470:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52032
2025-11-11T17:19:54.970156Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518795315591455:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:54.970914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518791020624074:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:54.970963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:54.971008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518791020624074:2049] [node 2] ICP22 created new session: [5:7571518795315591471:2049]
2025-11-11T17:19:54.971044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518795315591471:2049] [node 2] ICS09 handshake done sender: [5:7571518795315591455:2118] self: [5:7571518795315591458:0] peer: [2:7571518794281928261:0] socket: 386 qp: -1
2025-11-11T17:19:54.971078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518795315591471:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:54.971111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:54.969664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518796378887484:2066] [node 5] ICS09 handshake done sender: [3:7571518796378887463:2114] self: [3:7571518796378887462:0] peer: [5:7571518795315591459:0] socket: 408 qp: -1
2025-11-11T17:19:54.969728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518796378887484:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:54.970268Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:19:54.977707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:54.986247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518796378887410:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(68138212ns) {HandleHandshakeStatus:329(68135301ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(482ns) SwitchToState:178(325ns) ProcessPendingSessionEvents:495(119934ns) {ForwardSessionEventToSession:612(64761ns) ForwardSessionEventToSession:612(6102ns) ForwardSessionEventToSession:612(3337ns) ForwardSessionEventToSession:612(8594ns) ForwardSessionEventToSession:612(2527ns) ForwardSessionEventToSession:612(3689ns) ForwardSessionEventToSession:612(6796ns) ForwardSessionEventToSession:612(3041ns) ForwardSessionEventToSession:612(3054ns) ForwardSessionEventToSession:612(4974ns)}}}
2025-11-11T17:19:54.986856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:54.987102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518794281928237:2066] [node 5] ICP22 created new session: [2:7571518794281928271:2066]
2025-11-11T17:19:54.987143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518794281928271:2066] [node 5] ICS09 handshake done sender: [2:7571518794281928262:2120] self: [2:7571518794281928261:0] peer: [5:7571518795315591458:0] socket: 409 qp: -1
2025-11-11T17:19:54.987178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518794281928271:2066] [node 5] ICS10 traffic start
2025-11-11T17:19:54.987245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:54.987682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518794281928237:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(23383337ns) {HandleHandshakeStatus:329(23381458ns) {DropIncomingHandshake:441(30517ns) DropOutgoingHandshake:456(425ns) SwitchToState:178(537ns) ProcessPendingSessionEvents:495(66178ns) {ForwardSessionEventToSession:612(31848ns) ForwardSessionEventToSession:612(2539ns) ForwardSessionEventToSession:612(4488ns) ForwardSessionEventToSession:612(2481ns) ForwardSessionEventToSession:612(4091ns) ForwardSessionEventToSession:612(3727ns) ForwardSessionEventToSession:612(2666ns) ForwardSessionEventToSession:612(5153ns) ForwardSessionEventToSession:612(1743ns) ForwardSessionEventToSession:612(1672ns)}}}
2025-11-11T17:19:55.080561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:55.080822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518796378887469:2048] [node 1] ICS01 socket: 366 reason# EndOfStream
2025-11-11T17:19:55.080864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518792083920090:2048] [node 1] ICP30 unregister session Session# [3:7571518796378887469:2048] VirtualId# [3:7571518796378887466:0]
2025-11-11T17:19:55.080898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518796378887469:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.080970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:55.081014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:55.081710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:55.081950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518794281928264:2048] [node 1] ICS01 socket: 391 reason# EndOfStream
2025-11-11T17:19:55.081984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518789986960872:2048] [node 1] ICP30 unregister session Session# [2:7571518794281928264:2048] VirtualId# [2:7571518794281928248:0]
2025-11-11T17:19:55.082013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518794281928264:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.082073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:55.082113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:55.085775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:55.086077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518794492870179:2048] [node 1] ICS07 socket disconnect 360 reason# EndOfStream
2025-11-11T17:19:55.086104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518794492870179:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.086164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:55.086196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518794492870179:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:55.086771Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Couldn't connect to any resolved address
2025-11-11T17:19:55.087006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518790197902800:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:55.087050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518794492870179:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:55.087080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518790197902800:2048] [node 1] ICP30 unregister session Session# [4:7571518794492870179:2048] VirtualId# [4:7571518794492870167:0]
2025-11-11T17:19:55.087128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:55.087164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518790197902800:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:55.087179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:55.103164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518790197902800:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:55.093614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:55.093917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518795315591465:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-11-11T17:19:55.093957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518791020624073:2048] [node 1] ICP30 unregister session Session# [5:7571518795315591465:2048] VirtualId# [5:7571518795315591460:0]
2025-11-11T17:19:55.093994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518795315591465:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.094087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:55.094123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:55.103781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:55.104034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518795455339162:2048] [node 1] ICS01 socket: 382 reason# EndOfStream
2025-11-11T17:19:55.104080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518791160371764:2048] [node 1] ICP30 unregister session Session# [6:7571518795455339162:2048] VirtualId# [6:7571518795455339156:0]
2025-11-11T17:19:55.104111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518795455339162:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.104177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:55.104221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:55.201906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:55.202131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518796378887485:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-11-11T17:19:55.202177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518792083920091:2049] [node 2] ICP30 unregister session Session# [3:7571518796378887485:2049] VirtualId# [3:7571518796378887481:0]
2025-11-11T17:19:55.202211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518796378887485:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.202280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:55.202315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:55.209790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:55.210010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518794492870198:2049] [node 2] ICS01 socket: 367 reason# EndOfStream
2025-11-11T17:19:55.210051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518790197902801:2049] [node 2] ICP30 unregister session Session# [4:7571518794492870198:2049] VirtualId# [4:7571518794492870184:0]
2025-11-11T17:19:55.210085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518794492870198:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.210169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:55.210213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:55.210307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:55.210511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518795315591471:2049] [node 2] ICS01 socket: 386 reason# EndOfStream
2025-11-11T17:19:55.210538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518791020624074:2049] [node 2] ICP30 unregister session Session# [5:7571518795315591471:2049] VirtualId# [5:7571518795315591458:0]
2025-11-11T17:19:55.210562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518795315591471:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.210618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:55.210663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:55.213679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:55.213921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518795455339153:2049] [node 2] ICS07 socket disconnect 377 reason# EndOfStream
2025-11-11T17:19:55.213965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518795455339153:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.214066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:55.214117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518795455339153:2049] [node 2] ICS15 start handshake
2025-11-11T17:19:55.214974Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Couldn't connect to any resolved address
2025-11-11T17:19:55.215192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518791160371765:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:55.215227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518795455339153:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:55.215260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518791160371765:2049] [node 2] ICP30 unregister session Session# [6:7571518795455339153:2049] VirtualId# [6:7571518795455339140:0]
2025-11-11T17:19:55.215294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:55.215319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518791160371765:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:55.215334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:55.229676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518791160371765:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:19:55.309779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:55.310023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518795455339159:2050] [node 3] ICS01 socket: 374 reason# EndOfStream
2025-11-11T17:19:55.310062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518791160371766:2050] [node 3] ICP30 unregister session Session# [6:7571518795455339159:2050] VirtualId# [6:7571518795455339148:0]
2025-11-11T17:19:55.310098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518795455339159:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.310175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:55.310214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:55.310293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:55.310471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518795315591468:2050] [node 3] ICS01 socket: 410 reason# EndOfStream
2025-11-11T17:19:55.310498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518791020624075:2050] [node 3] ICP30 unregister session Session# [5:7571518795315591468:2050] VirtualId# [5:7571518795315591459:0]
2025-11-11T17:19:55.310522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518795315591468:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.310564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:55.310593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:55.313695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:55.313971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518794492870196:2050] [node 3] ICS01 socket: 387 reason# EndOfStream
2025-11-11T17:19:55.314025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518790197902802:2050] [node 3] ICP30 unregister session Session# [4:7571518794492870196:2050] VirtualId# [4:7571518794492870189:0]
2025-11-11T17:19:55.314066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518794492870196:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.314172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:55.314244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:55.389773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:55.389972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518795315591461:2066] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-11-11T17:19:55.390015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518795315591388:2066] [node 4] ICP30 unregister session Session# [5:7571518795315591461:2066] VirtualId# [5:7571518795315591453:0]
2025-11-11T17:19:55.390044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518795315591461:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.390134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:55.390176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:55.390261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:55.390432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518795455339161:2069] [node 4] ICS01 socket: 394 reason# EndOfStream
2025-11-11T17:19:55.390465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518795455339117:2069] [node 4] ICP30 unregister session Session# [6:7571518795455339161:2069] VirtualId# [6:7571518795455339141:0]
2025-11-11T17:19:55.390491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518795455339161:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.390541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:55.390576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:55.461738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:55.461961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518795455339145:2073] [node 5] ICS01 socket: 378 reason# EndOfStream
2025-11-11T17:19:55.462001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518795455339116:2073] [node 5] ICP30 unregister session Session# [6:7571518795455339145:2073] VirtualId# [6:7571518795455339135:0]
2025-11-11T17:19:55.462031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518795455339145:2073] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:55.462133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:55.462174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:55.576987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518800271611763:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:55.577035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518800271611764:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:55.596265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518800717365093:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:55.596312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518800717365094:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:55.645827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518799582185352:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:55.645872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518799582185353:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:55.708839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518799772549591:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:55.708880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518799772549592:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:55.708906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518799772549593:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:55.805996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518798953835330:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:55.806045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518798953835331:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:55.806092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518798953835332:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:55.912848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518798689442515:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:55.912892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518798689442516:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:55.912926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518798689442517:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:56.057337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518805012332407:2064] [node 5] ICP01 ready to work
2025-11-11T17:19:56.086299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518804067516903:2062] [node 5] ICP01 ready to work
2025-11-11T17:19:56.087996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518803877152663:2061] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.180492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518803877152666:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.184650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518803877152667:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-11T17:19:56.228101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518800271611764:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(167317467ns) {Configure:104(167313883ns) {ProcessConfigured:135(66991112ns) {StartInitialHandshake:157(66988056ns) {DropHandshakes:469(2158ns) {DropIncomingHandshake:441(529ns) DropOutgoingHandshake:456(374ns)} GenerateSessionVirtualId:988(1052ns) PrepareNewSessionHandshake:476(4679ns) {SwitchToState:178(2928ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.201775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518803248802645:2062] [node 6] ICP01 ready to work
2025-11-11T17:19:56.199424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518803877152663:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(111114305ns) {Configure:104(111110211ns) {ProcessConfigured:135(37488487ns) {StartInitialHandshake:157(37485573ns) {DropHandshakes:469(11928414ns) {DropIncomingHandshake:441(449ns) DropOutgoingHandshake:456(3060ns)} GenerateSessionVirtualId:988(3088ns) PrepareNewSessionHandshake:476(6080ns) {SwitchToState:178(3219ns)}}}}}
2025-11-11T17:19:56.257803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518805012332426:2065] [node 6] ICP01 ready to work
2025-11-11T17:19:56.237275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518804566579109:2076] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.268673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516911:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.387357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518805012332461:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50951
2025-11-11T17:19:56.387471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518805012332462:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50951
2025-11-11T17:19:56.281975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518802984409831:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:56.254664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518803248802653:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.234032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518803877152670:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-11T17:19:56.373618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518804067516910:2060] [node 6] ICP01 ready to work
2025-11-11T17:19:56.441566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518805012332459:2071] [node 4] ICP01 ready to work
2025-11-11T17:19:56.239991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518804566579096:2077] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.418096Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152670:2066] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.266327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518803248802656:2073] [node 4] ICP01 ready to work
2025-11-11T17:19:56.409385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518799772549593:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(348670608ns) {Configure:104(348666023ns) {ProcessConfigured:135(35452501ns) {StartInitialHandshake:157(35449806ns) {DropHandshakes:469(2773ns) {DropIncomingHandshake:441(979ns) DropOutgoingHandshake:456(525ns)} GenerateSessionVirtualId:988(824ns) PrepareNewSessionHandshake:476(5120ns) {SwitchToState:178(2658ns)}}}}}
2025-11-11T17:19:56.462153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518804067516903:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(375533956ns) {Configure:104(375528998ns) {ProcessConfigured:135(48307493ns) {StartInitialHandshake:157(48304726ns) {DropHandshakes:469(2317ns) {DropIncomingHandshake:441(562ns) DropOutgoingHandshake:456(394ns)} GenerateSessionVirtualId:988(647ns) PrepareNewSessionHandshake:476(4375ns) {SwitchToState:178(1679ns)}}}}}
2025-11-11T17:19:56.465662Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518804067516911:2067] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.278885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518804566579123:2078] [node 4] ICP01 ready to work
2025-11-11T17:19:56.507358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518798689442515:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(158659412ns) {Configure:104(158654830ns) {ProcessConfigured:135(106826ns) {StartInitialHandshake:157(103413ns) {DropHandshakes:469(3081ns) {DropIncomingHandshake:441(450ns) DropOutgoingHandshake:456(774ns)} GenerateSessionVirtualId:988(895ns) PrepareNewSessionHandshake:476(1831ns) {SwitchToState:178(910ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.489796Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332462:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.418455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518803877152719:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-11T17:19:56.520676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518798689442516:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(366649112ns) {Configure:104(366644705ns) {ProcessConfigured:135(77021ns) {StartInitialHandshake:157(74486ns) {DropHandshakes:469(13204ns) {DropIncomingHandshake:441(11200ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(775ns) PrepareNewSessionHandshake:476(1658ns) {SwitchToState:178(594ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.470879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518798953835330:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(344953121ns) {Configure:104(344941874ns) {ProcessConfigured:135(60762ns) {StartInitialHandshake:157(57932ns) {DropHandshakes:469(2184ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(821ns) PrepareNewSessionHandshake:476(1386ns) {SwitchToState:178(511ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.324671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518804566579128:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5155
2025-11-11T17:19:56.490019Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332461:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.464614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518803877152696:2075] [node 4] ICP01 ready to work
2025-11-11T17:19:56.471130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518803248802696:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
2025-11-11T17:19:56.375648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518804566579096:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(73393019ns) {Configure:104(73387685ns) {ProcessConfigured:135(68649ns) {StartInitialHandshake:157(66442ns) {DropHandshakes:469(1367ns) {DropIncomingHandshake:441(294ns) DropOutgoingHandshake:456(271ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:476(1645ns) {SwitchToState:178(566ns)}}}}}
2025-11-11T17:19:56.534062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409882:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.490079Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518803248802653:2071] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.514191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518805012332460:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50951
2025-11-11T17:19:56.485700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518803877152722:2078] [node 5] ICP01 ready to work
2025-11-11T17:19:56.485713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152666:2063] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.494971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518799582185352:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(368992521ns) {Configure:104(368988309ns) {ProcessConfigured:135(108766ns) {StartInitialHandshake:157(105057ns) {DropHandshakes:469(3650ns) {DropIncomingHandshake:441(988ns) DropOutgoingHandshake:456(513ns)} GenerateSessionVirtualId:988(945ns) PrepareNewSessionHandshake:476(2445ns) {SwitchToState:178(960ns)}}}}}
2025-11-11T17:19:56.447303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518804566579109:2076] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(149335816ns) {Configure:104(149328511ns) {ProcessConfigured:135(72559ns) {StartInitialHandshake:157(70185ns) {DropHandshakes:469(2098ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(376ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:476(1504ns) {SwitchToState:178(515ns)}}}}}
2025-11-11T17:19:56.549863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518802984409881:2078] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.563091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518803248802645:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(261198249ns) {Configure:104(225217401ns) {ProcessConfigured:135(78039ns) {StartInitialHandshake:157(75651ns) {DropHandshakes:469(1850ns) {DropIncomingHandshake:441(274ns) DropOutgoingHandshake:456(374ns)} GenerateSessionVirtualId:988(568ns) PrepareNewSessionHandshake:476(1757ns) {SwitchToState:178(664ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.495192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518799582185352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.522797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518805012332464:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50951
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.465782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579128:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.542030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332415:2069] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.584097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518798689442517:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(429884573ns) {Configure:104(429881755ns) {ProcessConfigured:135(92121ns) {StartInitialHandshake:157(89580ns) {DropHandshakes:469(2282ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:476(1235ns) {SwitchToState:178(476ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.511723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152667:2064] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.495547Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579117:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.653651Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518803248802696:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.636855Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518802984409882:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.495575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518804566579117:2102] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.516604Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518804566579117:2102] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:56.522017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518804566579139:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5155
2025-11-11T17:19:56.522071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518804566579140:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5155
2025-11-11T17:19:56.574056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518804566579123:2078] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(176319488ns) {Configure:104(176310848ns) {ProcessConfigured:135(22954695ns) {StartInitialHandshake:157(22951454ns) {DropHandshakes:469(2249ns) {DropIncomingHandshake:441(459ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:476(4486ns) {SwitchToState:178(2034ns)}}}}}
2025-11-11T17:19:56.621936Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579140:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.621997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518800271611764:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.635877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579139:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.636386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518804566579145:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5155
2025-11-11T17:19:56.641783Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579119:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.641817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518804566579119:2103] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.641920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:56.641964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518800271611764:2049] [node 3] ICP22 created new session: [1:7571518804566579144:2049]
2025-11-11T17:19:56.642013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518804566579144:2049] [node 3] ICS09 handshake done sender: [1:7571518804566579117:2102] self: [1:7571518804566579094:0] peer: [3:7571518803877152726:0] socket: 369 qp: -1
2025-11-11T17:19:56.642042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518804566579144:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:56.642086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:56.642476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518800271611764:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(20311025ns) {HandleHandshakeStatus:329(20307988ns) {DropIncomingHandshake:441(407ns) DropOutgoingHandshake:456(4271ns) SwitchToState:178(606ns) ProcessPendingSessionEvents:495(91050ns) {ForwardSessionEventToSession:612(45602ns) ForwardSessionEventToSession:612(3172ns) ForwardSessionEventToSession:612(5989ns) ForwardSessionEventToSession:612(2566ns) ForwardSessionEventToSession:612(3761ns) ForwardSessionEventToSession:612(6491ns) ForwardSessionEventToSession:612(3008ns) ForwardSessionEventToSession:612(5331ns) ForwardSessionEventToSession:612(2099ns) ForwardSessionEventToSession:612(3825ns)}}}
2025-11-11T17:19:56.642586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518800271611764:2049] [node 3] ICP18 handshake (actor [1:7571518804566579139:2116]) is too old
2025-11-11T17:19:56.681796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518804566579138:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5155
2025-11-11T17:19:56.681886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579133:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.681915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518804566579133:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.682422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579136:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.682438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518804566579136:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.683290Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518804566579119:2103] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.516264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518799582185353:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.542065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518805012332415:2069] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.653966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518798953835330:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.697713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579145:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.542922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332460:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.551147Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518805012332415:2069] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:56.581657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518800717365094:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(33006138ns) {IncomingHandshake:261(6442843ns) {DropIncomingHandshake:441(1036ns) PrepareNewSessionHandshake:476(1828ns) {SwitchToState:178(615ns)} GenerateSessionVirtualId:988(866ns) IssueIncomingHandshakeReply:193(14529ns)}}
2025-11-11T17:19:56.601659Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332464:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.602213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518800717365093:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(491485617ns) {Configure:104(491474611ns) {ProcessConfigured:135(53487ns) {StartInitialHandshake:157(50497ns) {DropHandshakes:469(2105ns) {DropIncomingHandshake:441(529ns) DropOutgoingHandshake:456(564ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:476(1685ns) {SwitchToState:178(649ns)}}}}}
2025-11-11T17:19:56.602446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518800717365093:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.665859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518805012332459:2071] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(158900614ns) {Configure:104(158894614ns) {ProcessConfigured:135(68940ns) {StartInitialHandshake:157(66347ns) {DropHandshakes:469(2416ns) {DropIncomingHandshake:441(434ns) DropOutgoingHandshake:456(440ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:476(1226ns) {SwitchToState:178(502ns)}}}}}
2025-11-11T17:19:56.666583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518800717365094:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.666656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:56.681709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518800717365094:2049] [node 3] ICP22 created new session: [2:7571518805012332474:2049]
2025-11-11T17:19:56.701949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518805012332476:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50951
2025-11-11T17:19:56.705695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518805012332474:2049] [node 3] ICS09 handshake done sender: [2:7571518805012332415:2069] self: [2:7571518805012332414:0] peer: [3:7571518803877152728:0] socket: 363 qp: -1
2025-11-11T17:19:56.706932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518805012332426:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(289072343ns) {Configure:104(289063789ns) {ProcessConfigured:135(9020401ns) {StartInitialHandshake:157(9017679ns) {DropHandshakes:469(2174ns) {DropIncomingHandshake:441(542ns) DropOutgoingHandshake:456(693ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:476(2136ns) {SwitchToState:178(771ns)}}}}}
2025-11-11T17:19:56.707074Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332410:2066] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.707096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518805012332410:2066] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.708177Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518805012332460:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:56.713731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518805012332474:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:56.713780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:56.714186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518800717365093:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.714257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:56.714306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518800717365093:2048] [node 1] ICP22 created new session: [2:7571518805012332481:2048]
2025-11-11T17:19:56.714344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518805012332481:2048] [node 1] ICS09 handshake done sender: [2:7571518805012332460:2112] self: [2:7571518805012332469:0] peer: [1:7571518804566579118:0] socket: 374 qp: -1
2025-11-11T17:19:56.714371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518805012332481:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:56.714395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.522519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518803877152729:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-11T17:19:56.651610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409887:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.702209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.744342Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518805012332410:2066] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:56.657479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518798689442517:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.700428Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518804566579136:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:56.693984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518798953835331:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(432005466ns) {Configure:104(431996766ns) {ProcessConfigured:135(12098229ns) {StartInitialHandshake:157(12094199ns) {DropHandshakes:469(2439ns) {DropIncomingHandshake:441(465ns) DropOutgoingHandshake:456(395ns)} GenerateSessionVirtualId:988(605ns) PrepareNewSessionHandshake:476(3337ns) {SwitchToState:178(1461ns)}}}}}
2025-11-11T17:19:56.694220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518798953835331:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.745425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518803248802708:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
2025-11-11T17:19:56.744812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518804067516910:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(269433326ns) {Configure:104(269427019ns) {ProcessConfigured:135(15465683ns) {StartInitialHandshake:157(15462815ns) {DropHandshakes:469(1902ns) {DropIncomingHandshake:441(341ns) DropOutgoingHandshake:456(380ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:476(2448ns) {SwitchToState:178(915ns)}}}}}
2025-11-11T17:19:56.614217Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152719:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.614540Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518803877152670:2066] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:56.614999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518803877152732:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-11T17:19:56.634044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518803877152733:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-11T17:19:56.634122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518799582185352:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.634214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:56.634261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518799582185352:2048] [node 1] ICP22 created new session: [3:7571518803877152734:2048]
2025-11-11T17:19:56.634317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518803877152734:2048] [node 1] ICS09 handshake done sender: [3:7571518803877152670:2066] self: [3:7571518803877152726:0] peer: [1:7571518804566579094:0] socket: 371 qp: -1
2025-11-11T17:19:56.634354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518803877152734:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:56.634410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:56.634862Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518803877152666:2063] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:56.649865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518799582185353:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.649919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:56.649967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518799582185353:2049] [node 2] ICP22 created new session: [3:7571518803877152736:2049]
2025-11-11T17:19:56.650023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518803877152736:2049] [node 2] ICS09 handshake done sender: [3:7571518803877152666:2063] self: [3:7571518803877152728:0] peer: [2:7571518805012332414:0] socket: 364 qp: -1
2025-11-11T17:19:56.650055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518803877152736:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:56.650087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:56.665541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518803877152696:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(100413497ns) {Configure:104(100408022ns) {ProcessConfigured:135(47502996ns) {StartInitialHandshake:157(47499740ns) {DropHandshakes:469(3154ns) {DropIncomingHandshake:441(649ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:476(2615ns) {SwitchToState:178(423ns)}}}}}
2025-11-11T17:19:56.674615Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152668:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.674636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518803877152668:2065] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.676802Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518803877152668:2065] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:56.677022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518803877152663:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.677073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:56.677119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518803877152663:2061] [node 6] ICP22 created new session: [3:7571518803877152741:2061]
2025-11-11T17:19:56.677162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518803877152741:2061] [node 6] ICS09 handshake done sender: [3:7571518803877152668:2065] self: [3:7571518803877152665:0] peer: [6:7571518802984409886:0] socket: 367 qp: -1
2025-11-11T17:19:56.677194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518803877152741:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:56.677227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:56.681831Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152733:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.697069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518803877152663:2061] [node 6] ICP18 handshake (actor [3:7571518803877152733:2120]) is too old
2025-11-11T17:19:56.777682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152731:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.777743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518803877152731:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.745966Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518803248802696:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:56.714048Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579138:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.721856Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518804566579133:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:56.755016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518805012332407:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.754153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516968:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.674240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518802984409887:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.733708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518804566579109:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.765953Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518804067516965:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.766102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518799772549593:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.777861Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518804067516968:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.778037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518799772549591:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.802128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516971:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
2025-11-11T17:19:56.802209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516972:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
2025-11-11T17:19:56.755059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.733775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:56.805830Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518803877152731:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:56.746260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518798953835330:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.674393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518798689442515:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.755104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518805012332407:2064] [node 5] ICP22 created new session: [2:7571518805012332484:2064]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.844649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518803877152696:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.746319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:56.733829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518804566579109:2076] [node 6] ICP22 created new session: [1:7571518804566579153:2076]
2025-11-11T17:19:56.733888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518804566579153:2076] [node 6] ICS09 handshake done sender: [1:7571518804566579136:2114] self: [1:7571518804566579135:0] peer: [6:7571518802984409888:0] socket: 400 qp: -1
2025-11-11T17:19:56.733925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518804566579153:2076] [node 6] ICS10 traffic start
2025-11-11T17:19:56.818404Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518804067516972:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.706638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409889:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.755151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518805012332484:2064] [node 5] ICS09 handshake done sender: [2:7571518805012332410:2066] self: [2:7571518805012332409:0] peer: [5:7571518803248802707:0] socket: 360 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.746361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518798953835330:2048] [node 1] ICP22 created new session: [5:7571518803248802710:2048]
2025-11-11T17:19:56.727780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518802984409831:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(353700775ns) {Configure:104(353694082ns) {ProcessConfigured:135(13795863ns) {StartInitialHandshake:157(13792960ns) {DropHandshakes:469(1895ns) {DropIncomingHandshake:441(447ns) DropOutgoingHandshake:456(436ns)} GenerateSessionVirtualId:988(550ns) PrepareNewSessionHandshake:476(3424ns) {SwitchToState:178(1507ns)}}}}}
2025-11-11T17:19:56.828946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
2025-11-11T17:19:56.733962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:56.755186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518805012332484:2064] [node 5] ICS10 traffic start
2025-11-11T17:19:56.863775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:56.890577Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:56.893870Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8980) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:56.746409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518803248802710:2048] [node 1] ICS09 handshake done sender: [5:7571518803248802696:2111] self: [5:7571518803248802704:0] peer: [1:7571518804566579132:0] socket: 380 qp: -1
2025-11-11T17:19:56.748656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518804566579096:2077] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.746443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518803248802710:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:56.746472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:56.873767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518803248802709:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
2025-11-11T17:19:56.878254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518803248802713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
2025-11-11T17:19:56.894218Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518803248802653:2071] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:56.897843Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518803248802703:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.897879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518803248802703:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.927244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518798953835331:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.927336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:56.927376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518798953835331:2049] [node 2] ICP22 created new session: [5:7571518803248802716:2049]
2025-11-11T17:19:56.927412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518803248802716:2049] [node 2] ICS09 handshake done sender: [5:7571518803248802653:2071] self: [5:7571518803248802707:0] peer: [2:7571518805012332409:0] socket: 368 qp: -1
2025-11-11T17:19:56.927444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518803248802716:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:56.927485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:56.928429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518803248802714:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
2025-11-11T17:19:56.928480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518803248802715:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
2025-11-11T17:19:56.728123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409891:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.863833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518803877152696:2075] [node 4] ICP22 created new session: [3:7571518803877152748:2075]
2025-11-11T17:19:56.895339Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518804067516968:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:56.902228Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518804067516965:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:56.925941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518799772549591:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8980) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518804067516968:2117] held: no
2025-11-11T17:19:56.926038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518799772549591:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.926083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:56.926127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518799772549591:2048] [node 1] ICP22 created new session: [4:7571518804067516976:2048]
2025-11-11T17:19:56.926186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518804067516976:2048] [node 1] ICS09 handshake done sender: [4:7571518804067516968:2117] self: [4:7571518804067516970:0] peer: [1:7571518804566579141:0] socket: 392 qp: -1
2025-11-11T17:19:56.926227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518804067516976:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:56.926258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:56.926484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516973:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.755217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:56.954421Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518803248802714:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.954750Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518803248802703:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:56.748705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:56.738116Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.881853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518803877152722:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(288145655ns) {Configure:104(288144455ns) {ProcessConfigured:135(16307177ns) {StartInitialHandshake:157(12122190ns) {DropHandshakes:469(2560ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(5465ns) PrepareNewSessionHandshake:476(3683ns) {SwitchToState:178(1356ns)}} IncomingHandshake:261(4156082ns) {DropIncomingHandshake:441(308ns) PrepareNewSessionHandshake:476(300ns) {SwitchToState:178(124ns)} GenerateSessionVirtualId:988(344ns) IssueIncomingHandshakeReply:193(1954ns)}}}}
2025-11-11T17:19:56.945882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518799772549593:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518804067516965:2115] held: no
2025-11-11T17:19:56.756582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518804566579096:2077] [node 5] ICP22 created new session: [1:7571518804566579156:2077]
2025-11-11T17:19:56.961675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518803248802715:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.889670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518803877152748:2075] [node 4] ICS09 handshake done sender: [3:7571518803877152731:2118] self: [3:7571518803877152730:0] peer: [4:7571518804067516969:0] socket: 385 qp: -1
2025-11-11T17:19:56.799046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518800717365094:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(140934529ns) {HandleHandshakeStatus:329(136636782ns) {DropIncomingHandshake:441(20767ns) DropOutgoingHandshake:456(699ns) SwitchToState:178(235ns) ProcessPendingSessionEvents:495(9518578ns) {ForwardSessionEventToSession:612(9465361ns) ForwardSessionEventToSession:612(8942ns) ForwardSessionEventToSession:612(5281ns) ForwardSessionEventToSession:612(2953ns) ForwardSessionEventToSession:612(5628ns) ForwardSessionEventToSession:612(2848ns) ForwardSessionEventToSession:612(3241ns) ForwardSessionEventToSession:612(7401ns) ForwardSessionEventToSession:612(3353ns) ForwardSessionEventToSession:612(4393ns)}}}
2025-11-11T17:19:56.754439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518802984409881:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(203923040ns) {Configure:104(203915364ns) {ProcessConfigured:135(40580658ns) {StartInitialHandshake:157(40577566ns) {DropHandshakes:469(7953804ns) {DropIncomingHandshake:441(1940ns) DropOutgoingHandshake:456(253ns)} GenerateSessionVirtualId:988(2282ns) PrepareNewSessionHandshake:476(2919ns) {SwitchToState:178(1263ns)}}}}}
2025-11-11T17:19:56.785634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518800271611763:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(28035869ns) {IncomingHandshake:261(11869571ns) {DropIncomingHandshake:441(970ns) PrepareNewSessionHandshake:476(4381ns) {SwitchToState:178(2402ns)} GenerateSessionVirtualId:988(511ns) IssueIncomingHandshakeReply:193(5226ns)}}
2025-11-11T17:19:56.785727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518800271611763:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.789643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518804566579156:2077] [node 5] ICS09 handshake done sender: [1:7571518804566579133:2113] self: [1:7571518804566579132:0] peer: [5:7571518803248802704:0] socket: 377 qp: -1
2025-11-11T17:19:56.797082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518804566579156:2077] [node 5] ICS10 traffic start
2025-11-11T17:19:56.797644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:56.805692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:56.805703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518800271611763:2048] [node 2] ICP22 created new session: [1:7571518804566579158:2048]
2025-11-11T17:19:56.817233Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518804566579142:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.817274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518804566579142:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.818493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518804566579158:2048] [node 2] ICS09 handshake done sender: [1:7571518804566579119:2103] self: [1:7571518804566579118:0] peer: [2:7571518805012332469:0] socket: 370 qp: -1
2025-11-11T17:19:56.829617Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518804566579142:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:56.830106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518804566579158:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:56.853013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518804566579096:2077] [node 5] ICP35 event processing took too much time PendingConnection:278(104152691ns) {HandleHandshakeStatus:329(104150807ns) {DropIncomingHandshake:441(11845ns) DropOutgoingHandshake:456(372ns) SwitchToState:178(352ns) ProcessPendingSessionEvents:495(8220528ns) {ForwardSessionEventToSession:612(8167708ns) ForwardSessionEventToSession:612(11014ns) ForwardSessionEventToSession:612(3154ns) ForwardSessionEventToSession:612(5191ns) ForwardSessionEventToSession:612(3197ns) ForwardSessionEventToSession:612(2383ns) ForwardSessionEventToSession:612(4351ns) ForwardSessionEventToSession:612(2087ns) ForwardSessionEventToSession:612(3164ns) ForwardSessionEventToSession:612(5923ns)}}}
2025-11-11T17:19:56.854367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:56.857883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518804566579123:2078] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.874453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:56.882035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518804566579123:2078] [node 4] ICP22 created new session: [1:7571518804566579162:2078]
2025-11-11T17:19:56.894633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518800271611763:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(108699549ns) {HandleHandshakeStatus:329(108697545ns) {DropIncomingHandshake:441(3422367ns) DropOutgoingHandshake:456(1699ns) SwitchToState:178(840ns) ProcessPendingSessionEvents:495(12420724ns) {ForwardSessionEventToSession:612(12365938ns) ForwardSessionEventToSession:612(13225ns) ForwardSessionEventToSession:612(3375ns) ForwardSessionEventToSession:612(3287ns) ForwardSessionEventToSession:612(5056ns) ForwardSessionEventToSession:612(5680ns) ForwardSessionEventToSession:612(4894ns) ForwardSessionEventToSession:612(2560ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(4405ns)}}}
2025-11-11T17:19:56.917890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518804566579162:2078] [node 4] ICS09 handshake done sender: [1:7571518804566579142:2118] self: [1:7571518804566579141:0] peer: [4:7571518804067516970:0] socket: 408 qp: -1
2025-11-11T17:19:56.929960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518804566579162:2078] [node 4] ICS10 traffic start
2025-11-11T17:19:56.945700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.890474Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332477:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.949714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518804067516973:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.977833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518803248802713:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.978233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518803248802645:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.981961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518803248802656:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.985972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518798953835332:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.773728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518798689442517:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518802984409882:2112] held: no
2025-11-11T17:19:56.893845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518803877152748:2075] [node 4] ICS10 traffic start
2025-11-11T17:19:56.909820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:56.974001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518803877152696:2075] [node 4] ICP35 event processing took too much time PendingConnection:278(136098509ns) {HandleHandshakeStatus:329(136091007ns) {DropIncomingHandshake:441(16489ns) DropOutgoingHandshake:456(384ns) SwitchToState:178(365ns) ProcessPendingSessionEvents:495(15365959ns) {ForwardSessionEventToSession:612(15308073ns) ForwardSessionEventToSession:612(7038ns) ForwardSessionEventToSession:612(10854ns) ForwardSessionEventToSession:612(3674ns) ForwardSessionEventToSession:612(9473ns) ForwardSessionEventToSession:612(3188ns) ForwardSessionEventToSession:612(2521ns) ForwardSessionEventToSession:612(5088ns) ForwardSessionEventToSession:612(2231ns) ForwardSessionEventToSession:612(2374ns)}}}
2025-11-11T17:19:56.789643Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518802984409887:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:56.798306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409895:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.798371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409896:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.825661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518798689442515:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.829744Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8980) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:56.842744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:56.842802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518798689442515:2048] [node 1] ICP22 created new session: [6:7571518802984409898:2048]
2025-11-11T17:19:56.842841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518802984409898:2048] [node 1] ICS09 handshake done sender: [6:7571518802984409887:2114] self: [6:7571518802984409888:0] peer: [1:7571518804566579135:0] socket: 401 qp: -1
2025-11-11T17:19:56.842872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518802984409898:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:56.842910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:56.843356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518798689442515:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17525515ns) {HandleHandshakeStatus:329(17523281ns) {DropIncomingHandshake:441(775ns) DropOutgoingHandshake:456(7934ns) SwitchToState:178(426ns) ProcessPendingSessionEvents:495(94522ns) {ForwardSessionEventToSession:612(53323ns) ForwardSessionEventToSession:612(3060ns) ForwardSessionEventToSession:612(5049ns) ForwardSessionEventToSession:612(2195ns) ForwardSessionEventToSession:612(2001ns) ForwardSessionEventToSession:612(4630ns) ForwardSessionEventToSession:612(3593ns) ForwardSessionEventToSession:612(5108ns) ForwardSessionEventToSession:612(5045ns) ForwardSessionEventToSession:612(2714ns)}}}
2025-11-11T17:19:56.843451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518798689442515:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:56.849856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518798689442517:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(37600762ns) {HandleHandshakeStatus:395(37597019ns) {LogHandshakeFail:472(15318131ns) {UpdateErrorStateLog:544(8944ns)} DropOutgoingHandshake:456(3125ns) ProcessPendingSessionEvents:495(2223954ns) {DropSessionEvent:512(20117ns) DropSessionEvent:512(6259ns) DropSessionEvent:512(5581ns) DropSessionEvent:512(5462ns) DropSessionEvent:512(11880ns) DropSessionEvent:512(4524ns) DropSessionEvent:512(6152ns) DropSessionEvent:512(5577ns) DropSessionEvent:512(6270ns) DropSessionEvent:512(4392ns)}}}
2025-11-11T17:19:56.849955Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518802984409882:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:56.861807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518798689442517:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.861863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518802984409895:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.861997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518798689442516:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.873801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518802984409896:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.873927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518802984409881:2078] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:56.889842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:56.890859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518798689442517:2050] [node 3] ICP22 created new session: [6:7571518802984409903:2050]
2025-11-11T17:19:56.929776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518802984409903:2050] [node 3] ICS09 handshake done sender: [6:7571518802984409882:2112] self: [6:7571518802984409886:0] peer: [3:7571518803877152665:0] socket: 386 qp: -1
2025-11-11T17:19:56.939547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409904:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.941909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518802984409903:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:56.941959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:56.951174Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518802984409895:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:56.951602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518798689442516:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:56.951679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:56.951727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518798689442516:2049] [node 2] ICP22 created new session: [6:7571518802984409906:2049]
2025-11-11T17:19:56.951823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518802984409906:2049] [node 2] ICS09 handshake done sender: [6:7571518802984409895:2119] self: [6:7571518802984409900:0] peer: [2:7571518805012332475:0] socket: 410 qp: -1
2025-11-11T17:19:56.951852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518802984409906:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:56.951941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:56.961799Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:56.962069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518798689442517:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(100168402ns) {HandleHandshakeStatus:329(100166395ns) {DropIncomingHandshake:441(724ns) DropOutgoingHandshake:456(725ns) SwitchToState:178(1016ns) ProcessPendingSessionEvents:495(1317ns)}}
2025-11-11T17:19:56.962127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518798689442516:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:56.974827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409908:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.974962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518802984409909:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:56.989812Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518802984409896:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:56.998285Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518802984409908:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:57.005930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518802984409831:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:57.010511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518802984409881:2078] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:57.019026Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24728) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:56.954094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518799772549592:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:57.023802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:57.010593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:56.890509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518805012332477:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:57.041796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518802984409881:2078] [node 5] ICP22 created new session: [6:7571518807279377207:2078]
2025-11-11T17:19:57.021679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518803877152745:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:57.021721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518803877152745:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:57.023899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518803248802645:2062] [node 6] ICP22 created new session: [5:7571518807543770018:2062]
2025-11-11T17:19:56.914739Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518805012332477:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:57.048906Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518803877152745:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:56.982685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518799772549593:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(52242136ns) {HandleHandshakeStatus:395(52239845ns) {LogHandshakeFail:472(12905ns) {UpdateErrorStateLog:544(5249ns)} DropOutgoingHandshake:456(2821ns) ProcessPendingSessionEvents:495(66917ns) {DropSessionEvent:512(10412ns) DropSessionEvent:512(5629ns) DropSessionEvent:512(8847ns) DropSessionEvent:512(7223ns) DropSessionEvent:512(4486ns) DropSessionEvent:512(5657ns) DropSessionEvent:512(5713ns) DropSessionEvent:512(4811ns) DropSessionEvent:512(5733ns) DropSessionEvent:512(4614ns)}}}
2025-11-11T17:19:56.914966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518805012332426:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:57.047053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518807279377207:2078] [node 5] ICS09 handshake done sender: [6:7571518802984409896:2120] self: [6:7571518802984409901:0] peer: [5:7571518803248802702:0] socket: 411 qp: -1
2025-11-11T17:19:57.047184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518804566579123:2078] [node 4] ICP35 event processing took too much time PendingConnection:278(163790735ns) {HandleHandshakeStatus:329(163788783ns) {DropIncomingHandshake:441(12062ns) DropOutgoingHandshake:456(342ns) SwitchToState:178(537ns) ProcessPendingSessionEvents:495(59905801ns) {ForwardSessionEventToSession:612(25771615ns) ForwardSessionEventToSession:612(2563865ns) ForwardSessionEventToSession:612(3373ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(8088572ns) ForwardSessionEventToSession:612(7884767ns) ForwardSessionEventToSession:612(9953035ns) ForwardSessionEventToSession:612(4569ns) ForwardSessionEventToSession:612(3278ns) ForwardSessionEventToSession:612(5605469ns)}}}
2025-11-11T17:19:57.033639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518807543770018:2062] [node 6] ICS09 handshake done sender: [5:7571518803248802703:2115] self: [5:7571518803248802702:0] peer: [6:7571518802984409901:0] socket: 395 qp: -1
2025-11-11T17:19:57.049181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518803877152722:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:56.997646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518804067516981:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22567
2025-11-11T17:19:57.049285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:57.049326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518803877152722:2078] [node 5] ICP22 created new session: [3:7571518808172120047:2078]
2025-11-11T17:19:57.049367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518808172120047:2078] [node 5] ICS09 handshake done sender: [3:7571518803877152745:2121] self: [3:7571518803877152744:0] peer: [5:7571518803248802721:0] socket: 417 qp: -1
2025-11-11T17:19:57.049392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518808172120047:2078] [node 5] ICS10 traffic start
2025-11-11T17:19:57.049429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:57.037995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518807543770019:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:56.915021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:56.998130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518799772549593:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:57.047107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518807279377207:2078] [node 5] ICS10 traffic start
2025-11-11T17:19:57.122115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518803877152722:2078] [node 5] ICP35 event processing took too much time PendingConnection:278(72741184ns) {HandleHandshakeStatus:329(72734920ns) {DropIncomingHandshake:441(16147ns) DropOutgoingHandshake:456(326ns) SwitchToState:178(684ns) ProcessPendingSessionEvents:495(4197999ns) {ForwardSessionEventToSession:612(4146547ns) ForwardSessionEventToSession:612(6200ns) ForwardSessionEventToSession:612(6241ns) ForwardSessionEventToSession:612(3237ns) ForwardSessionEventToSession:612(5996ns) ForwardSessionEventToSession:612(5042ns) ForwardSessionEventToSession:612(3459ns) ForwardSessionEventToSession:612(7392ns) ForwardSessionEventToSession:612(2805ns) ForwardSessionEventToSession:612(1985ns)}}}
2025-11-11T17:19:56.915070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518805012332426:2065] [node 6] ICP22 created new session: [2:7571518805012332488:2065]
2025-11-11T17:19:56.915116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518805012332488:2065] [node 6] ICS09 handshake done sender: [2:7571518805012332477:2119] self: [2:7571518805012332475:0] peer: [6:7571518802984409900:0] socket: 409 qp: -1
2025-11-11T17:19:56.915148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518805012332488:2065] [node 6] ICS10 traffic start
2025-11-11T17:19:56.915177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:56.978285Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518805012332472:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:56.978320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518805012332472:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:56.996150Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518805012332472:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:57.001749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518805012332459:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:57.001811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:57.001860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518805012332459:2071] [node 4] ICP22 created new session: [2:7571518809307299787:2071]
2025-11-11T17:19:57.001908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518809307299787:2071] [node 4] ICS09 handshake done sender: [2:7571518805012332472:2117] self: [2:7571518805012332471:0] peer: [4:7571518804067516980:0] socket: 413 qp: -1
2025-11-11T17:19:57.001942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518809307299787:2071] [node 4] ICS10 traffic start
2025-11-11T17:19:57.001977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:57.002912Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:57.003683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518799772549592:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518804067516973:2120] held: no
2025-11-11T17:19:57.009825Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518804067516962:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:57.009863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518804067516962:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:57.013674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:57.018996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518799772549593:2050] [node 3] ICP22 created new session: [4:7571518808362484278:2050]
2025-11-11T17:19:57.023665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518804067516967:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:57.023698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518804067516967:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:57.030108Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518804067516962:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:57.030210Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518804067516973:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:57.046636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518808362484278:2050] [node 3] ICS09 handshake done sender: [4:7571518804067516965:2115] self: [4:7571518804067516969:0] peer: [3:7571518803877152730:0] socket: 388 qp: -1
2025-11-11T17:19:57.047216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518799772549592:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:57.047256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:57.047480Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518804067516967:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:57.053875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518808362484278:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:57.060385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518804067516903:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:57.061712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:57.077731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518799772549592:2049] [node 2] ICP22 created new session: [4:7571518808362484281:2049]
2025-11-11T17:19:57.077825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:57.089798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518804067516903:2062] [node 5] ICP22 created new session: [4:7571518808362484283:2062]
2025-11-11T17:19:57.089860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518808362484283:2062] [node 5] ICS09 handshake done sender: [4:7571518804067516962:2113] self: [4:7571518804067516957:0] peer: [5:7571518803248802720:0] socket: 361 qp: -1
2025-11-11T17:19:57.089901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518808362484283:2062] [node 5] ICS10 traffic start
2025-11-11T17:19:57.089943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:57.090321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518804067516903:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(30030521ns) {HandleHandshakeStatus:329(29774965ns) {DropIncomingHandshake:441(12108ns) DropOutgoingHandshake:456(179ns) SwitchToState:178(103ns) ProcessPendingSessionEvents:495(96215ns) {ForwardSessionEventToSession:612(47167ns) ForwardSessionEventToSession:612(6027ns) ForwardSessionEventToSession:612(2136ns) ForwardSessionEventToSession:612(4936ns) ForwardSessionEventToSession:612(2141ns) ForwardSessionEventToSession:612(2304ns) ForwardSessionEventToSession:612(5227ns) ForwardSessionEventToSession:612(5547ns) ForwardSessionEventToSession:612(7195ns) ForwardSessionEventToSession:612(3931ns)}}}
2025-11-11T17:19:57.090421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518804067516910:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:57.090491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:57.090532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518804067516910:2060] [node 6] ICP22 created new session: [4:7571518808362484285:2060]
2025-11-11T17:19:57.090569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518808362484285:2060] [node 6] ICS09 handshake done sender: [4:7571518804067516967:2116] self: [4:7571518804067516966:0] peer: [6:7571518807279377206:0] socket: 381 qp: -1
2025-11-11T17:19:57.090593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518808362484285:2060] [node 6] ICS10 traffic start
2025-11-11T17:19:57.090615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:57.101708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518808362484281:2049] [node 2] ICS09 handshake done sender: [4:7571518804067516973:2120] self: [4:7571518804067516980:0] peer: [2:7571518805012332471:0] socket: 414 qp: -1
2025-11-11T17:19:57.101759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518808362484281:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:57.101819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:57.102007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518799772549592:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(54719698ns) {HandleHandshakeStatus:329(54716528ns) {DropIncomingHandshake:441(618ns) DropOutgoingHandshake:456(392ns) SwitchToState:178(423ns) ProcessPendingSessionEvents:495(832ns)}}
2025-11-11T17:19:57.113420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518799772549593:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(110655393ns) {HandleHandshakeStatus:329(110653712ns) {DropIncomingHandshake:441(1485ns) DropOutgoingHandshake:456(671ns) SwitchToState:178(616ns) ProcessPendingSessionEvents:495(759ns)}}
2025-11-11T17:19:57.048436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518807279377208:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-11T17:19:57.048917Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518802984409908:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:57.049172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518802984409831:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:57.057814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:57.057867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518802984409831:2061] [node 4] ICP22 created new session: [6:7571518807279377209:2061]
2025-11-11T17:19:57.057909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518807279377209:2061] [node 4] ICS09 handshake done sender: [6:7571518802984409908:2122] self: [6:7571518807279377206:0] peer: [4:7571518804067516966:0] socket: 421 qp: -1
2025-11-11T17:19:57.057940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518807279377209:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:57.057981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:57.070033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:57.112744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518802984409881:2078] [node 5] ICP35 event processing took too much time PendingConnection:278(102005576ns) {HandleHandshakeStatus:329(102003179ns) {DropIncomingHandshake:441(661ns) DropOutgoingHandshake:456(4244ns) SwitchToState:178(346ns) ProcessPendingSessionEvents:495(14574905ns) {ForwardSessionEventToSession:612(14526450ns) ForwardSessionEventToSession:612(5209ns) ForwardSessionEventToSession:612(6537ns) ForwardSessionEventToSession:612(4570ns) ForwardSessionEventToSession:612(3398ns) ForwardSessionEventToSession:612(2355ns) ForwardSessionEventToSession:612(5192ns) ForwardSessionEventToSession:612(2491ns) ForwardSessionEventToSession:612(5561ns) ForwardSessionEventToSession:612(2358ns)}}}
2025-11-11T17:19:57.112805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518802984409881:2078] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:57.041875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518807543770018:2062] [node 6] ICS10 traffic start
2025-11-11T17:19:57.041921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:57.042323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518803248802645:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(63908589ns) {HandleHandshakeStatus:329(63906596ns) {DropIncomingHandshake:441(13559ns) DropOutgoingHandshake:456(371ns) SwitchToState:178(239ns) ProcessPendingSessionEvents:495(106424ns) {ForwardSessionEventToSession:612(49304ns) ForwardSessionEventToSession:612(5275ns) ForwardSessionEventToSession:612(2909ns) ForwardSessionEventToSession:612(2603ns) ForwardSessionEventToSession:612(16589ns) ForwardSessionEventToSession:612(2583ns) ForwardSessionEventToSession:612(6118ns) ForwardSessionEventToSession:612(2671ns) ForwardSessionEventToSession:612(4339ns) ForwardSessionEventToSession:612(4681ns)}}}
2025-11-11T17:19:57.042794Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518803248802715:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:57.058466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518803248802656:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:57.058554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:57.058598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518803248802656:2073] [node 4] ICP22 created new session: [5:7571518807543770021:2073]
2025-11-11T17:19:57.058640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518807543770021:2073] [node 4] ICS09 handshake done sender: [5:7571518803248802715:2121] self: [5:7571518803248802720:0] peer: [4:7571518804067516957:0] socket: 419 qp: -1
2025-11-11T17:19:57.058667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518807543770021:2073] [node 4] ICS10 traffic start
2025-11-11T17:19:57.058705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:57.085996Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:57.087042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518798953835332:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518803248802713:2119] held: no
2025-11-11T17:19:57.109885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518807543770023:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39008
2025-11-11T17:19:57.128057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518803248802713:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:57.131632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518798953835332:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:57.133511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:57.133571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518798953835332:2050] [node 3] ICP22 created new session: [5:7571518807543770024:2050]
2025-11-11T17:19:57.133640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518807543770024:2050] [node 3] ICS09 handshake done sender: [5:7571518803248802713:2119] self: [5:7571518803248802721:0] peer: [3:7571518803877152744:0] socket: 382 qp: -1
2025-11-11T17:19:57.133675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518807543770024:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:57.133721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:57.209734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:57.209989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518805012332481:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-11-11T17:19:57.210033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518800717365093:2048] [node 1] ICP30 unregister session Session# [2:7571518805012332481:2048] VirtualId# [2:7571518805012332469:0]
2025-11-11T17:19:57.210067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518805012332481:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.210143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:57.210182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:57.213689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:57.213909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518803877152734:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-11-11T17:19:57.213951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518799582185352:2048] [node 1] ICP30 unregister session Session# [3:7571518803877152734:2048] VirtualId# [3:7571518803877152726:0]
2025-11-11T17:19:57.213984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518803877152734:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.214069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:57.214110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:57.221746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:57.221967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518802984409898:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-11-11T17:19:57.222001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518798689442515:2048] [node 1] ICP30 unregister session Session# [6:7571518802984409898:2048] VirtualId# [6:7571518802984409888:0]
2025-11-11T17:19:57.222030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518802984409898:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.222088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:57.222124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:57.221828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:57.222035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518804067516976:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-11-11T17:19:57.222060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518799772549591:2048] [node 1] ICP30 unregister session Session# [4:7571518804067516976:2048] VirtualId# [4:7571518804067516970:0]
2025-11-11T17:19:57.222082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518804067516976:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.222140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:57.222172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:57.222201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:57.222378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518803248802710:2048] [node 1] ICS01 socket: 380 reason# EndOfStream
2025-11-11T17:19:57.222404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518798953835330:2048] [node 1] ICP30 unregister session Session# [5:7571518803248802710:2048] VirtualId# [5:7571518803248802704:0]
2025-11-11T17:19:57.222426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518803248802710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.222467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:57.222493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:57.337736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:57.337989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518803877152736:2049] [node 2] ICS01 socket: 364 reason# EndOfStream
2025-11-11T17:19:57.338107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518799582185353:2049] [node 2] ICP30 unregister session Session# [3:7571518803877152736:2049] VirtualId# [3:7571518803877152728:0]
2025-11-11T17:19:57.338156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518803877152736:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.338254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:57.338294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:57.345739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:57.345758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:57.345754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:57.345948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518808362484281:2049] [node 2] ICS01 socket: 414 reason# EndOfStream
2025-11-11T17:19:57.345947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518803248802716:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-11-11T17:19:57.345943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518802984409906:2049] [node 2] ICS01 socket: 410 reason# EndOfStream
2025-11-11T17:19:57.345981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518798953835331:2049] [node 2] ICP30 unregister session Session# [5:7571518803248802716:2049] VirtualId# [5:7571518803248802707:0]
2025-11-11T17:19:57.345985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518798689442516:2049] [node 2] ICP30 unregister session Session# [6:7571518802984409906:2049] VirtualId# [6:7571518802984409900:0]
2025-11-11T17:19:57.346009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518803248802716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.345978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518799772549592:2049] [node 2] ICP30 unregister session Session# [4:7571518808362484281:2049] VirtualId# [4:7571518804067516980:0]
2025-11-11T17:19:57.346009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518802984409906:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.346012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518808362484281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.346084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:57.346084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:57.346072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:57.346119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:57.346118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:57.346110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:57.441703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:57.441936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518802984409903:2050] [node 3] ICS01 socket: 386 reason# EndOfStream
2025-11-11T17:19:57.441988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518798689442517:2050] [node 3] ICP30 unregister session Session# [6:7571518802984409903:2050] VirtualId# [6:7571518802984409886:0]
2025-11-11T17:19:57.442023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518802984409903:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.442099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:57.442133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:57.442478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:57.442661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518808362484278:2050] [node 3] ICS01 socket: 388 reason# EndOfStream
2025-11-11T17:19:57.442689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518799772549593:2050] [node 3] ICP30 unregister session Session# [4:7571518808362484278:2050] VirtualId# [4:7571518804067516969:0]
2025-11-11T17:19:57.442712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518808362484278:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.442756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:57.442781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:57.449698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:57.449883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518807543770024:2050] [node 3] ICS01 socket: 382 reason# EndOfStream
2025-11-11T17:19:57.449916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518798953835332:2050] [node 3] ICP30 unregister session Session# [5:7571518807543770024:2050] VirtualId# [5:7571518803248802721:0]
2025-11-11T17:19:57.449951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518807543770024:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.450043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:57.450087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:57.583863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:57.584095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518807543770021:2073] [node 4] ICS01 socket: 419 reason# EndOfStream
2025-11-11T17:19:57.584130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518803248802656:2073] [node 4] ICP30 unregister session Session# [5:7571518807543770021:2073] VirtualId# [5:7571518803248802720:0]
2025-11-11T17:19:57.584168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518807543770021:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.584243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:57.584278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:57.585357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:57.585555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518807279377209:2061] [node 4] ICS01 socket: 421 reason# EndOfStream
2025-11-11T17:19:57.585597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518802984409831:2061] [node 4] ICP30 unregister session Session# [6:7571518807279377209:2061] VirtualId# [6:7571518807279377206:0]
2025-11-11T17:19:57.585621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518807279377209:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.585686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:57.585724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:57.645699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:19:57.645926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518807279377207:2078] [node 5] ICS01 socket: 411 reason# EndOfStream
2025-11-11T17:19:57.645973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518802984409881:2078] [node 5] ICP30 unregister session Session# [6:7571518807279377207:2078] VirtualId# [6:7571518802984409901:0]
2025-11-11T17:19:57.646006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518807279377207:2078] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:57.646115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:19:57.646158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:57.837642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518810082646535:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:57.837690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518810082646536:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:57.849875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518809218050929:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:57.849913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518809218050930:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:57.953822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518806793978089:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:57.953869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518806793978090:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:57.997864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518809201382403:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:57.997903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518809201382404:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:57.997931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518809201382405:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:58.039969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518811318110707:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:58.040021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518811318110708:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:58.040820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518811318110709:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:58.103725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518814377613844:2058] [node 5] ICP01 ready to work
2025-11-11T17:19:58.139703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518814377613848:2061] [node 6] ICP01 ready to work
2025-11-11T17:19:58.140058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518813513018240:2059] [node 5] ICP01 ready to work
2025-11-11T17:19:58.154444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518811088945405:2060] [node 6] ICP01 ready to work
2025-11-11T17:19:58.154719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518811088945406:2063] [node 5] ICP01 ready to work
2025-11-11T17:19:58.294939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518811318110708:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(222352663ns) {Configure:104(222348032ns) {ProcessConfigured:135(61048141ns) {StartInitialHandshake:157(61044439ns) {DropHandshakes:469(2394ns) {DropIncomingHandshake:441(506ns) DropOutgoingHandshake:456(593ns)} GenerateSessionVirtualId:988(921ns) PrepareNewSessionHandshake:476(7754ns) {SwitchToState:178(3587ns)}}}}}
2025-11-11T17:19:58.269852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518813513018241:2061] [node 6] ICP01 ready to work
2025-11-11T17:19:58.439570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518809218050930:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(169452311ns) {Configure:104(169447484ns) {ProcessConfigured:135(55241ns) {StartInitialHandshake:157(52838ns) {DropHandshakes:469(2003ns) {DropIncomingHandshake:441(527ns) DropOutgoingHandshake:456(352ns)} GenerateSessionVirtualId:988(891ns) PrepareNewSessionHandshake:476(4233ns) {SwitchToState:178(474ns)}}}}}
2025-11-11T17:19:58.443088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518813513018294:2075] [node 4] ICP01 ready to work
2025-11-11T17:19:58.295386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518811318110750:2061] [node 4] ICP01 ready to work
2025-11-11T17:19:58.349927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518811088945406:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(177181894ns) {Configure:104(177177876ns) {ProcessConfigured:135(115952521ns) {StartInitialHandshake:157(115949359ns) {DropHandshakes:469(35951926ns) {DropIncomingHandshake:441(4957ns) DropOutgoingHandshake:456(420ns)} GenerateSessionVirtualId:988(6345ns) PrepareNewSessionHandshake:476(5556ns) {SwitchToState:178(2437ns)}}}}}
2025-11-11T17:19:58.351986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518811088945430:2068] [node 4] ICP01 ready to work
2025-11-11T17:19:58.295743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518811318110751:2070] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.306303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518814094105369:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:58.306393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518814094105370:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:58.306446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518814094105371:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:58.307076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518814094105417:2061] [node 5] ICP01 ready to work
2025-11-11T17:19:58.307340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518814094105418:2066] [node 4] ICP01 ready to work
2025-11-11T17:19:58.490518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518814094105370:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(181078423ns) {Configure:104(181074680ns) {ProcessConfigured:135(74235ns) {StartInitialHandshake:157(71346ns) {DropHandshakes:469(2019ns) {DropIncomingHandshake:441(550ns) DropOutgoingHandshake:456(392ns)} GenerateSessionVirtualId:988(1183ns) PrepareNewSessionHandshake:476(1557ns) {SwitchToState:178(485ns)}}}}}
2025-11-11T17:19:58.531276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518814377613844:2058] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(384341228ns) {Configure:104(384335868ns) {ProcessConfigured:135(57840ns) {StartInitialHandshake:157(54892ns) {DropHandshakes:469(4446ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(355ns)} GenerateSessionVirtualId:988(656ns) PrepareNewSessionHandshake:476(1344ns) {SwitchToState:178(467ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.545893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518813513018294:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(92861596ns) {Configure:104(92858515ns) {ProcessConfigured:135(27977762ns) {StartInitialHandshake:157(27975091ns) {DropHandshakes:469(2218ns) {DropIncomingHandshake:441(666ns) DropOutgoingHandshake:456(457ns)} GenerateSessionVirtualId:988(1187ns) PrepareNewSessionHandshake:476(6002ns) {SwitchToState:178(2622ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.549724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105444:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.509626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110785:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.547990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518811318110785:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.555413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518811318110709:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.566243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110787:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.573998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518810082646535:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(41175041ns) {Configure:104(41171593ns) {ProcessConfigured:135(63739ns) {StartInitialHandshake:157(61924ns) {DropHandshakes:469(2242ns) {DropIncomingHandshake:441(503ns) DropOutgoingHandshake:456(450ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:476(1585ns) {SwitchToState:178(641ns)}}}}}
2025-11-11T17:19:58.574401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518814377613904:2066] [node 4] ICP01 ready to work
2025-11-11T17:19:58.546281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518813513018295:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59420
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.585880Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018295:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.586199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518813513018305:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59420
2025-11-11T17:19:58.584808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518809201382405:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(467967109ns) {Configure:104(467962850ns) {ProcessConfigured:135(35364082ns) {StartInitialHandshake:157(35361244ns) {DropHandshakes:469(2308ns) {DropIncomingHandshake:441(414ns) DropOutgoingHandshake:456(395ns)} GenerateSessionVirtualId:988(860ns) PrepareNewSessionHandshake:476(4348ns) {SwitchToState:178(393ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.578005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518811088945427:2083] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.606191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110788:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.578036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518811088945427:2083] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.629769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518814377613904:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33184691ns) {Configure:104(33182113ns) {ProcessConfigured:135(32125989ns) {StartInitialHandshake:157(32124292ns) {DropHandshakes:469(994ns) {DropIncomingHandshake:441(102ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(5720ns) {SwitchToState:178(2115ns)}}}}}
2025-11-11T17:19:58.630049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518814377613898:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48738
2025-11-11T17:19:58.630243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518814377613899:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48738
2025-11-11T17:19:58.630683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518814377613905:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48738
2025-11-11T17:19:58.633443Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518814094105444:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.618058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518813496349764:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.658818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518813513018307:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59420
2025-11-11T17:19:58.658894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518813513018308:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59420
2025-11-11T17:19:58.579015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518811088945464:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9518
2025-11-11T17:19:58.579060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518811088945465:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9518
2025-11-11T17:19:58.590371Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518811088945427:2083] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:58.654237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518811088945406:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.654326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:58.654372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518811088945406:2063] [node 5] ICP22 created new session: [3:7571518811088945467:2063]
2025-11-11T17:19:58.654417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518811088945467:2063] [node 5] ICS09 handshake done sender: [3:7571518811088945427:2083] self: [3:7571518811088945412:0] peer: [5:7571518811318110786:0] socket: 371 qp: -1
2025-11-11T17:19:58.654456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518811088945467:2063] [node 5] ICS10 traffic start
2025-11-11T17:19:58.654489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:58.618439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518813496349765:2070] [node 6] ICP01 ready to work
2025-11-11T17:19:58.676970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018287:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.677007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518813513018287:2106] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.678681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518813513018287:2106] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.681947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518814094105371:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.682059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105448:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.682176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105449:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.680999Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018308:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.681089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018305:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.681127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518809218050930:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.681197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:58.681239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518809218050930:2049] [node 3] ICP22 created new session: [2:7571518813513018310:2049]
2025-11-11T17:19:58.681281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518813513018310:2049] [node 3] ICS09 handshake done sender: [2:7571518813513018287:2106] self: [2:7571518813513018286:0] peer: [3:7571518811088945470:0] socket: 377 qp: -1
2025-11-11T17:19:58.681309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518813513018310:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:58.681336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.669968Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518811088945464:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.671295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518806793978090:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.678246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518811088945471:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9518
2025-11-11T17:19:58.678437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518811088945465:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.679223Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518811088945464:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:58.679380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518806793978090:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.679447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:58.679490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518806793978090:2049] [node 2] ICP22 created new session: [3:7571518811088945473:2049]
2025-11-11T17:19:58.679532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518811088945473:2049] [node 2] ICS09 handshake done sender: [3:7571518811088945464:2115] self: [3:7571518811088945470:0] peer: [2:7571518813513018286:0] socket: 378 qp: -1
2025-11-11T17:19:58.679567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518811088945473:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:58.679614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:58.686344Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518811088945408:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.686367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518811088945408:2065] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.693779Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518811088945408:2065] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:58.690288Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518811318110785:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:58.694094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105451:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.706584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518809218050930:2049] [node 3] ICP18 handshake (actor [2:7571518813513018308:2118]) is too old
2025-11-11T17:19:58.707970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613905:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.708063Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613898:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.708131Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613899:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.713399Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018307:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.713513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518809218050929:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.702219Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518814094105444:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.702398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518814094105371:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.702448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:58.702498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518814094105371:2050] [node 3] ICP22 created new session: [6:7571518814094105452:2050]
2025-11-11T17:19:58.702539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518814094105452:2050] [node 3] ICS09 handshake done sender: [6:7571518814094105444:2115] self: [6:7571518814094105450:0] peer: [3:7571518811088945407:0] socket: 373 qp: -1
2025-11-11T17:19:58.702570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518814094105452:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:58.702619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:58.710723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518811088945405:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.710801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:58.710846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518811088945405:2060] [node 6] ICP22 created new session: [3:7571518811088945477:2060]
2025-11-11T17:19:58.710894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518811088945477:2060] [node 6] ICS09 handshake done sender: [3:7571518811088945408:2065] self: [3:7571518811088945407:0] peer: [6:7571518814094105450:0] socket: 370 qp: -1
2025-11-11T17:19:58.710928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518811088945477:2060] [node 6] ICS10 traffic start
2025-11-11T17:19:58.710962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:58.709823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518811318110709:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.709880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:58.709924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518811318110709:2050] [node 3] ICP22 created new session: [5:7571518811318110790:2050]
2025-11-11T17:19:58.714360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518811318110790:2050] [node 3] ICS09 handshake done sender: [5:7571518811318110785:2115] self: [5:7571518811318110786:0] peer: [3:7571518811088945412:0] socket: 374 qp: -1
2025-11-11T17:19:58.714401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518811318110790:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:58.714437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:58.714696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518814094105448:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.714771Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518814094105449:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.716229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518814094105370:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.716293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518814094105369:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.722296Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018293:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.722330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518813513018293:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.726731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105457:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.726927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110789:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.714943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518809201382403:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(93216507ns) {Configure:104(93213389ns) {ProcessConfigured:135(48404ns) {StartInitialHandshake:157(46376ns) {DropHandshakes:469(1461ns) {DropIncomingHandshake:441(271ns) DropOutgoingHandshake:456(215ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:476(680ns) {SwitchToState:178(127ns)}}}}}
2025-11-11T17:19:58.730756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349766:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.731233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349775:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.713923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518814377613916:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48738
2025-11-11T17:19:58.744260Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518814094105448:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.744556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518814094105370:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.744630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:58.744675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518814094105370:2049] [node 2] ICP22 created new session: [6:7571518814094105459:2049]
2025-11-11T17:19:58.744714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518814094105459:2049] [node 2] ICS09 handshake done sender: [6:7571518814094105448:2116] self: [6:7571518814094105455:0] peer: [2:7571518813513018292:0] socket: 394 qp: -1
2025-11-11T17:19:58.744747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518814094105459:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:58.744780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:58.745197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105458:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.737922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613903:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.751413Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518811318110787:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.750960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518811088945479:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9518
2025-11-11T17:19:58.731688Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518813513018293:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:58.759186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518813513018241:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.759245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:58.759315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518813513018241:2061] [node 6] ICP22 created new session: [2:7571518813513018318:2061]
2025-11-11T17:19:58.759363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518813513018318:2061] [node 6] ICS09 handshake done sender: [2:7571518813513018293:2110] self: [2:7571518813513018292:0] peer: [6:7571518814094105455:0] socket: 388 qp: -1
2025-11-11T17:19:58.759402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518813513018318:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:58.759444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:58.737958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518814377613903:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.738534Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613901:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.738550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518814377613901:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.739080Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613916:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.758241Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518814377613903:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.761980Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518813496349775:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.762448Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518811318110789:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.766517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105462:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.767182Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018289:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.767210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518813513018289:2107] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.767648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518813513018316:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59420
2025-11-11T17:19:58.767712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518813513018317:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59420
2025-11-11T17:19:58.762868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110792:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.763064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518811318110708:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.763131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518811318110707:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.763307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110793:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.765828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518811318110792:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.772509Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518814094105449:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:58.773127Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518814094105462:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.773174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518814094105369:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.773219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:58.773265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518814094105369:2048] [node 1] ICP22 created new session: [6:7571518814094105463:2048]
2025-11-11T17:19:58.773300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518814094105463:2048] [node 1] ICS09 handshake done sender: [6:7571518814094105449:2117] self: [6:7571518814094105456:0] peer: [1:7571518814377613900:0] socket: 395 qp: -1
2025-11-11T17:19:58.773329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518814094105463:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:58.773394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:58.774947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518814094105417:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.786159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518813496349766:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.786614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518809201382405:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.786734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349779:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.770338Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518814377613901:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:58.777510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105466:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.790237Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518811088945479:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.798251Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518813513018307:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:58.775248Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518811318110759:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.798474Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.802143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349782:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.802501Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613897:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.802532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518814377613897:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.802953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518810082646535:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.803012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:58.803053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518810082646535:2048] [node 2] ICP22 created new session: [1:7571518814377613920:2048]
2025-11-11T17:19:58.803101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518814377613920:2048] [node 2] ICS09 handshake done sender: [1:7571518814377613903:2114] self: [1:7571518814377613902:0] peer: [2:7571518813513018314:0] socket: 393 qp: -1
2025-11-11T17:19:58.803129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518814377613920:2048] [node 2] ICS10 traffic start
2025-11-11T17:19:58.803158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:58.803528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518814377613848:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.803609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:58.803647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518814377613848:2061] [node 6] ICP22 created new session: [1:7571518814377613922:2061]
2025-11-11T17:19:58.803684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518814377613922:2061] [node 6] ICS09 handshake done sender: [1:7571518814377613901:2113] self: [1:7571518814377613900:0] peer: [6:7571518814094105456:0] socket: 392 qp: -1
2025-11-11T17:19:58.803713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518814377613922:2061] [node 6] ICS10 traffic start
2025-11-11T17:19:58.803737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:58.775276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518811318110759:2092] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.776338Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518811318110759:2092] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:58.776548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518811318110751:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.776592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:58.776633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518811318110751:2070] [node 6] ICP22 created new session: [5:7571518811318110799:2070]
2025-11-11T17:19:58.776675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518811318110799:2070] [node 6] ICS09 handshake done sender: [5:7571518811318110759:2092] self: [5:7571518811318110758:0] peer: [6:7571518814094105465:0] socket: 419 qp: -1
2025-11-11T17:19:58.776706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518811318110799:2070] [node 6] ICS10 traffic start
2025-11-11T17:19:58.776762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:58.805743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518811318110793:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.815363Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518814377613897:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:58.815740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518814377613844:2058] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.815797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:58.815840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518814377613844:2058] [node 5] ICP22 created new session: [1:7571518814377613927:2058]
2025-11-11T17:19:58.815877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518814377613927:2058] [node 5] ICS09 handshake done sender: [1:7571518814377613897:2110] self: [1:7571518814377613896:0] peer: [5:7571518811318110796:0] socket: 391 qp: -1
2025-11-11T17:19:58.815905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518814377613927:2058] [node 5] ICS10 traffic start
2025-11-11T17:19:58.815938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:58.815144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349783:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.817855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518809218050929:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.817922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:58.817974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518809218050929:2048] [node 1] ICP22 created new session: [2:7571518813513018322:2048]
2025-11-11T17:19:58.818014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518813513018322:2048] [node 1] ICS09 handshake done sender: [2:7571518813513018307:2117] self: [2:7571518813513018314:0] peer: [1:7571518814377613902:0] socket: 397 qp: -1
2025-11-11T17:19:58.818044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518813513018322:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:58.818088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:58.819142Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518811088945432:2085] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.829578Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613907:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.829627Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518814377613907:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.831738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518814377613907:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:58.832060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518810082646536:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.832115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:58.832152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518810082646536:2049] [node 3] ICP22 created new session: [1:7571518814377613930:2049]
2025-11-11T17:19:58.832191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518814377613930:2049] [node 3] ICS09 handshake done sender: [1:7571518814377613907:2116] self: [1:7571518814377613906:0] peer: [3:7571518811088945485:0] socket: 398 qp: -1
2025-11-11T17:19:58.832222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518814377613930:2049] [node 3] ICS10 traffic start
2025-11-11T17:19:58.832247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.819198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518811088945432:2085] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.827536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518811088945480:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9518
2025-11-11T17:19:58.827929Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518811088945432:2085] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:58.828269Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518811088945480:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.828307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518811088945430:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.828377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:58.828420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518811088945430:2068] [node 4] ICP22 created new session: [3:7571518811088945483:2068]
2025-11-11T17:19:58.828458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518811088945483:2068] [node 4] ICS09 handshake done sender: [3:7571518811088945432:2085] self: [3:7571518811088945431:0] peer: [4:7571518813496349780:0] socket: 401 qp: -1
2025-11-11T17:19:58.828484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518811088945483:2068] [node 4] ICS10 traffic start
2025-11-11T17:19:58.828542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:58.829344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518806793978089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.833200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518811088945486:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9518
2025-11-11T17:19:58.833518Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518811088945480:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.837981Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518813496349783:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.838122Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518813496349779:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.838271Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11813) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:58.839626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518809201382405:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11813) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518813496349775:2116] held: no
2025-11-11T17:19:58.839982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518809201382403:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.840103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349784:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.840685Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518813496349775:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:19:58.840886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518809201382405:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.841878Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:58.840925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:19:58.826254Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518814094105462:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:58.822084Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018316:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.822168Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:58.823814Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518813513018289:2107] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:58.824110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518809218050929:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:58.824225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518813513018240:2059] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.824274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:58.824318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518813513018240:2059] [node 5] ICP22 created new session: [2:7571518813513018324:2059]
2025-11-11T17:19:58.824353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518813513018324:2059] [node 5] ICS09 handshake done sender: [2:7571518813513018289:2107] self: [2:7571518813513018288:0] peer: [5:7571518811318110795:0] socket: 382 qp: -1
2025-11-11T17:19:58.824381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518813513018324:2059] [node 5] ICS10 traffic start
2025-11-11T17:19:58.824411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:58.854829Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518813513018301:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.854853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518813513018301:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.855932Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518813513018301:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.856502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518813513018294:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.856572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:58.856613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518813513018294:2075] [node 4] ICP22 created new session: [2:7571518813513018328:2075]
2025-11-11T17:19:58.856654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518813513018328:2075] [node 4] ICS09 handshake done sender: [2:7571518813513018301:2115] self: [2:7571518813513018300:0] peer: [4:7571518813496349790:0] socket: 385 qp: -1
2025-11-11T17:19:58.842163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518806793978089:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.856684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518813513018328:2075] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.840964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518809201382405:2050] [node 3] ICP22 created new session: [4:7571518813496349788:2050]
2025-11-11T17:19:58.857866Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613909:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.857898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518814377613909:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.859467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518814377613932:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48738
2025-11-11T17:19:58.859716Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518814377613909:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:58.859919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518814377613904:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.859954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:58.859993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518814377613904:2066] [node 4] ICP22 created new session: [1:7571518814377613934:2066]
2025-11-11T17:19:58.860036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518814377613934:2066] [node 4] ICS09 handshake done sender: [1:7571518814377613909:2117] self: [1:7571518814377613908:0] peer: [4:7571518813496349785:0] socket: 423 qp: -1
2025-11-11T17:19:58.860065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518814377613934:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:58.860091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:58.840995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518813496349788:2050] [node 3] ICS09 handshake done sender: [4:7571518813496349775:2116] self: [4:7571518813496349780:0] peer: [3:7571518811088945431:0] socket: 407 qp: -1
2025-11-11T17:19:58.841027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518813496349788:2050] [node 3] ICS10 traffic start
2025-11-11T17:19:58.841054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:19:58.854483Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518813496349782:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.854598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518809201382404:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.856248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349791:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.861825Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7400) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:58.856741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:58.830683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518814094105417:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518814094105462:2121] held: no
2025-11-11T17:19:58.830770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518814094105417:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.830819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:58.830861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518814094105417:2061] [node 5] ICP22 created new session: [6:7571518814094105467:2061]
2025-11-11T17:19:58.830905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518814094105467:2061] [node 5] ICS09 handshake done sender: [6:7571518814094105462:2121] self: [6:7571518814094105465:0] peer: [5:7571518811318110758:0] socket: 420 qp: -1
2025-11-11T17:19:58.830932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518814094105467:2061] [node 5] ICS10 traffic start
2025-11-11T17:19:58.830980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:58.849764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:58.889767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518814377613932:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.849822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518806793978089:2048] [node 1] ICP22 created new session: [3:7571518811088945487:2048]
2025-11-11T17:19:58.849896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518811088945487:2048] [node 1] ICS09 handshake done sender: [3:7571518811088945480:2119] self: [3:7571518811088945485:0] peer: [1:7571518814377613906:0] socket: 376 qp: -1
2025-11-11T17:19:58.849930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518811088945487:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:58.849974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:19:58.850271Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:58.866548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518806793978089:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:19:58.882442Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518813496349782:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:58.892206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518814377613904:2066] [node 4] ICP18 handshake (actor [1:7571518814377613932:2119]) is too old
2025-11-11T17:19:58.890514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518809201382404:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7400) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518813496349782:2118] held: no
2025-11-11T17:19:58.890602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518809201382404:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.890673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:58.890719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518809201382404:2049] [node 2] ICP22 created new session: [4:7571518813496349793:2049]
2025-11-11T17:19:58.890761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518813496349793:2049] [node 2] ICS09 handshake done sender: [4:7571518813496349782:2118] self: [4:7571518813496349790:0] peer: [2:7571518813513018300:0] socket: 381 qp: -1
2025-11-11T17:19:58.890794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518813496349793:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:58.890828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:58.891821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518813496349792:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58637
2025-11-11T17:19:58.894092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518811318110707:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518811318110789:2118] held: no
2025-11-11T17:19:58.894257Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7400) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:58.895520Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:19:58.896558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518809201382403:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518813496349783:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.896826Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518813496349783:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:58.899168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105469:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.901782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518811318110750:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.921787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518811318110708:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7400) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518811318110787:2116] held: no
2025-11-11T17:19:58.922012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110801:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.922376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518809201382403:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.922464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:58.922513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518809201382403:2048] [node 1] ICP22 created new session: [4:7571518813496349796:2048]
2025-11-11T17:19:58.922681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518813496349796:2048] [node 1] ICS09 handshake done sender: [4:7571518813496349783:2119] self: [4:7571518813496349785:0] peer: [1:7571518814377613908:0] socket: 424 qp: -1
2025-11-11T17:19:58.922723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518813496349796:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:58.922764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.928325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518813496349770:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.928356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518813496349770:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.929266Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518814094105469:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.929394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518814094105418:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:19:58.925540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110802:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.926128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518811318110787:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:19:58.926344Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518811318110789:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:19:58.926528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518811318110708:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.926567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:19:58.926604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518811318110708:2049] [node 2] ICP22 created new session: [5:7571518811318110804:2049]
2025-11-11T17:19:58.926651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518811318110804:2049] [node 2] ICS09 handshake done sender: [5:7571518811318110787:2116] self: [5:7571518811318110795:0] peer: [2:7571518813513018288:0] socket: 383 qp: -1
2025-11-11T17:19:58.926687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518811318110804:2049] [node 2] ICS10 traffic start
2025-11-11T17:19:58.926742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:19:58.927020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518811318110707:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.927050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:19:58.927340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518811318110707:2048] [node 1] ICP22 created new session: [5:7571518811318110806:2048]
2025-11-11T17:19:58.927380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518811318110806:2048] [node 1] ICS09 handshake done sender: [5:7571518811318110789:2118] self: [5:7571518811318110796:0] peer: [1:7571518814377613896:0] socket: 403 qp: -1
2025-11-11T17:19:58.927404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518811318110806:2048] [node 1] ICS10 traffic start
2025-11-11T17:19:58.927430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.934403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518811318110808:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36412
2025-11-11T17:19:58.934609Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518813496349770:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:19:58.935127Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518811318110793:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:58.935395Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518813496349774:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:19:58.935417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518813496349774:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:19:58.936111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518813496349764:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.936184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:19:58.936221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518813496349764:2067] [node 5] ICP22 created new session: [4:7571518813496349799:2067]
2025-11-11T17:19:58.936262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518813496349799:2067] [node 5] ICS09 handshake done sender: [4:7571518813496349770:2113] self: [4:7571518813496349769:0] peer: [5:7571518811318110803:0] socket: 410 qp: -1
2025-11-11T17:19:58.936292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518813496349799:2067] [node 5] ICS10 traffic start
2025-11-11T17:19:58.936320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:19:58.937168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518811318110750:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.937255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:58.937292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518811318110750:2061] [node 4] ICP22 created new session: [5:7571518811318110809:2061]
2025-11-11T17:19:58.937324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518811318110809:2061] [node 4] ICS09 handshake done sender: [5:7571518811318110793:2120] self: [5:7571518811318110803:0] peer: [4:7571518813496349769:0] socket: 418 qp: -1
2025-11-11T17:19:58.937363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518811318110809:2061] [node 4] ICS10 traffic start
2025-11-11T17:19:58.937392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:19:58.943133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518814094105471:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32070
2025-11-11T17:19:58.943490Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518813496349774:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:19:58.943994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518814094105469:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:19:58.944219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518814094105418:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:19:58.944271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:19:58.944308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518814094105418:2066] [node 4] ICP22 created new session: [6:7571518814094105472:2066]
2025-11-11T17:19:58.944344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518814094105472:2066] [node 4] ICS09 handshake done sender: [6:7571518814094105469:2123] self: [6:7571518814094105470:0] peer: [4:7571518813496349773:0] socket: 363 qp: -1
2025-11-11T17:19:58.944378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518814094105472:2066] [node 4] ICS10 traffic start
2025-11-11T17:19:58.944410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:19:58.945837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518813496349765:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:19:58.945912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:19:58.945953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518813496349765:2070] [node 6] ICP22 created new session: [4:7571518813496349802:2070]
2025-11-11T17:19:58.945987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518813496349802:2070] [node 6] ICS09 handshake done sender: [4:7571518813496349774:2115] self: [4:7571518813496349773:0] peer: [6:7571518814094105470:0] socket: 362 qp: -1
2025-11-11T17:19:58.946017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518813496349802:2070] [node 6] ICS10 traffic start
2025-11-11T17:19:58.946041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:19:58.998558Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:19:58.998772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518813513018322:2048] [node 1] ICS01 socket: 397 reason# EndOfStream
2025-11-11T17:19:58.998812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518809218050929:2048] [node 1] ICP30 unregister session Session# [2:7571518813513018322:2048] VirtualId# [2:7571518813513018314:0]
2025-11-11T17:19:58.998858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518813513018322:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:58.998939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:58.998988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:59.003928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:19:59.004145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518811088945487:2048] [node 1] ICS07 socket disconnect 376 reason# EndOfStream
2025-11-11T17:19:59.004171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518811088945487:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.004235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:59.004271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518811088945487:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:59.007035Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:19:59.007975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518806793978089:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:19:59.008015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518811088945487:2048] [node 1] ICS15 start handshake
2025-11-11T17:19:59.008613Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Couldn't connect to any resolved address
2025-11-11T17:19:59.008822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518806793978089:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:19:59.008866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518811088945487:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:19:59.008899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518806793978089:2048] [node 1] ICP30 unregister session Session# [3:7571518811088945487:2048] VirtualId# [3:7571518811088945485:0]
2025-11-11T17:19:59.008946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:59.008993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518806793978089:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:59.009010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:19:59.013765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:19:59.013975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518811318110806:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-11-11T17:19:59.014014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518811318110707:2048] [node 1] ICP30 unregister session Session# [5:7571518811318110806:2048] VirtualId# [5:7571518811318110796:0]
2025-11-11T17:19:59.014045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518811318110806:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.014110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:59.014147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:59.017047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:19:59.017245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518814094105463:2048] [node 1] ICS01 socket: 395 reason# EndOfStream
2025-11-11T17:19:59.017930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518806793978089:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:19:59.017283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518814094105369:2048] [node 1] ICP30 unregister session Session# [6:7571518814094105463:2048] VirtualId# [6:7571518814094105456:0]
2025-11-11T17:19:59.017326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518814094105463:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.017406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:59.017446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:59.017531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:19:59.017749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518813496349796:2048] [node 1] ICS01 socket: 424 reason# EndOfStream
2025-11-11T17:19:59.017784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518809201382403:2048] [node 1] ICP30 unregister session Session# [4:7571518813496349796:2048] VirtualId# [4:7571518813496349785:0]
2025-11-11T17:19:59.017807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518813496349796:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.017885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:19:59.017934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:19:59.100926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:19:59.101145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518811088945473:2049] [node 2] ICS01 socket: 378 reason# EndOfStream
2025-11-11T17:19:59.101193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518806793978090:2049] [node 2] ICP30 unregister session Session# [3:7571518811088945473:2049] VirtualId# [3:7571518811088945470:0]
2025-11-11T17:19:59.101226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518811088945473:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.101302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:59.101345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:59.114821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:19:59.115045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518811318110804:2049] [node 2] ICS01 socket: 383 reason# EndOfStream
2025-11-11T17:19:59.115087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518811318110708:2049] [node 2] ICP30 unregister session Session# [5:7571518811318110804:2049] VirtualId# [5:7571518811318110795:0]
2025-11-11T17:19:59.115119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518811318110804:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.115202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:59.115260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:59.115671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:19:59.115890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518814094105459:2049] [node 2] ICS01 socket: 394 reason# EndOfStream
2025-11-11T17:19:59.115945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518814094105370:2049] [node 2] ICP30 unregister session Session# [6:7571518814094105459:2049] VirtualId# [6:7571518814094105455:0]
2025-11-11T17:19:59.115970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518814094105459:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.116020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:59.116049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:59.129745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:19:59.129953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518813496349793:2049] [node 2] ICS01 socket: 381 reason# EndOfStream
2025-11-11T17:19:59.129993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518809201382404:2049] [node 2] ICP30 unregister session Session# [4:7571518813496349793:2049] VirtualId# [4:7571518813496349790:0]
2025-11-11T17:19:59.130020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518813496349793:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.130119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:19:59.130157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:19:59.253794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:19:59.253994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518814094105452:2050] [node 3] ICS01 socket: 373 reason# EndOfStream
2025-11-11T17:19:59.254037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518814094105371:2050] [node 3] ICP30 unregister session Session# [6:7571518814094105452:2050] VirtualId# [6:7571518814094105450:0]
2025-11-11T17:19:59.254070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518814094105452:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.254151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:59.254198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:59.254289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:19:59.254440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518811318110790:2050] [node 3] ICS01 socket: 374 reason# EndOfStream
2025-11-11T17:19:59.254468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518811318110709:2050] [node 3] ICP30 unregister session Session# [5:7571518811318110790:2050] VirtualId# [5:7571518811318110786:0]
2025-11-11T17:19:59.254493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518811318110790:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.254542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:59.254577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:59.254652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:19:59.254801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518813496349788:2050] [node 3] ICS01 socket: 407 reason# EndOfStream
2025-11-11T17:19:59.254832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518809201382405:2050] [node 3] ICP30 unregister session Session# [4:7571518813496349788:2050] VirtualId# [4:7571518813496349780:0]
2025-11-11T17:19:59.254853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518813496349788:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.254894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:19:59.254923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:19:59.323936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:19:59.324135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518814094105472:2066] [node 4] ICS01 socket: 363 reason# EndOfStream
2025-11-11T17:19:59.323936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:19:59.324144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518811318110809:2061] [node 4] ICS01 socket: 418 reason# EndOfStream
2025-11-11T17:19:59.324181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518811318110750:2061] [node 4] ICP30 unregister session Session# [5:7571518811318110809:2061] VirtualId# [5:7571518811318110803:0]
2025-11-11T17:19:59.324210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518811318110809:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.324299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:59.324339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:19:59.324173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518814094105418:2066] [node 4] ICP30 unregister session Session# [6:7571518814094105472:2066] VirtualId# [6:7571518814094105470:0]
2025-11-11T17:19:59.324203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518814094105472:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:19:59.324269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:19:59.324305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:19:59.662770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518817503078028:2048] [node 2] ICP01 ready to work
2025-11-11T17:19:59.662819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518817503078029:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:59.718448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518814961563927:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:59.718488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518814961563928:2049] [node 3] ICP01 ready to work
2025-11-11T17:19:59.777953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518817128547848:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:59.777991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518817128547849:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:59.841897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518816198475780:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:59.841939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518816198475781:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:59.841967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518816198475782:2050] [node 3] ICP01 ready to work
2025-11-11T17:19:59.869925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518816719939776:2048] [node 1] ICP01 ready to work
2025-11-11T17:19:59.869972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518816719939777:2049] [node 2] ICP01 ready to work
2025-11-11T17:19:59.870000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518816719939778:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:00.101937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518819256531244:2064] [node 5] ICP01 ready to work
2025-11-11T17:20:00.098231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518817103000830:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:00.098356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518817103000831:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:00.098408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518817103000832:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:00.118782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518821397968139:2059] [node 4] ICP01 ready to work
2025-11-11T17:20:00.145650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518821798045352:2067] [node 4] ICP01 ready to work
2025-11-11T17:20:00.113638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518819256531246:2066] [node 6] ICP01 ready to work
2025-11-11T17:20:00.130342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518816198475780:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36295513ns) {Configure:104(30354146ns) {ProcessConfigured:135(9668288ns) {StartInitialHandshake:157(9665058ns) {DropHandshakes:469(3267ns) {DropIncomingHandshake:441(787ns) DropOutgoingHandshake:456(455ns)} GenerateSessionVirtualId:988(868ns) PrepareNewSessionHandshake:476(6023ns) {SwitchToState:178(1937ns)}}}}}
2025-11-11T17:20:00.137708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518820493443091:2061] [node 5] ICP01 ready to work
2025-11-11T17:20:00.162178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518821798045353:2068] [node 6] ICP01 ready to work
2025-11-11T17:20:00.162501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518821798045354:2069] [node 5] ICP01 ready to work
2025-11-11T17:20:00.158053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518819256531248:2067] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.197999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518819256531244:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35629632ns) {Configure:104(35624892ns) {ProcessConfigured:135(21958934ns) {StartInitialHandshake:157(21955341ns) {DropHandshakes:469(3490ns) {DropIncomingHandshake:441(690ns) DropOutgoingHandshake:456(918ns)} GenerateSessionVirtualId:988(813ns) PrepareNewSessionHandshake:476(5482ns) {SwitchToState:178(2129ns)}}}}}
2025-11-11T17:20:00.187224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518820493443091:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(38761493ns) {Configure:104(38754792ns) {ProcessConfigured:135(75827ns) {StartInitialHandshake:157(72381ns) {DropHandshakes:469(4106ns) {DropIncomingHandshake:441(1276ns) DropOutgoingHandshake:456(689ns)} GenerateSessionVirtualId:988(1234ns) PrepareNewSessionHandshake:476(2066ns) {SwitchToState:178(704ns)}}}}}
2025-11-11T17:20:00.177957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518821397968139:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(58683396ns) {Configure:104(58677798ns) {ProcessConfigured:135(11858403ns) {StartInitialHandshake:157(11855440ns) {DropHandshakes:469(4004ns) {DropIncomingHandshake:441(941ns) DropOutgoingHandshake:456(745ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:476(3464ns) {SwitchToState:178(1234ns)}}}}}
2025-11-11T17:20:00.171337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518817128547849:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(88416752ns) {Configure:104(83272134ns) {ProcessConfigured:135(80095ns) {StartInitialHandshake:157(74519ns) {DropHandshakes:469(3315ns) {DropIncomingHandshake:441(858ns) DropOutgoingHandshake:456(612ns)} GenerateSessionVirtualId:988(1240ns) PrepareNewSessionHandshake:476(1754ns) {SwitchToState:178(611ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.222984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518817503078029:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(156425527ns) {Configure:104(148790249ns) {ProcessConfigured:135(73610ns) {StartInitialHandshake:157(67007ns) {DropHandshakes:469(3603ns) {DropIncomingHandshake:441(815ns) DropOutgoingHandshake:456(581ns)} GenerateSessionVirtualId:988(1290ns) PrepareNewSessionHandshake:476(2266ns) {SwitchToState:178(656ns)}}}}}
2025-11-11T17:20:00.171820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518821423515161:2061] [node 6] ICP01 ready to work
2025-11-11T17:20:00.228239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518821798045363:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30587
2025-11-11T17:20:00.235496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518821798045367:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30587
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.187747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518820493443097:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
2025-11-11T17:20:00.266054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518821798045352:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(103202160ns) {Configure:104(103197638ns) {ProcessConfigured:135(7736849ns) {StartInitialHandshake:157(7733295ns) {DropHandshakes:469(7672622ns) {DropIncomingHandshake:441(818ns) DropOutgoingHandshake:456(1767ns)} GenerateSessionVirtualId:988(2634ns) PrepareNewSessionHandshake:476(2335ns) {SwitchToState:178(1188ns)}}}}}
2025-11-11T17:20:00.258957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518819256531246:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(100615642ns) {Configure:104(100609243ns) {ProcessConfigured:135(109557ns) {StartInitialHandshake:157(107147ns) {DropHandshakes:469(1639ns) {DropIncomingHandshake:441(434ns) DropOutgoingHandshake:456(504ns)} GenerateSessionVirtualId:988(932ns) PrepareNewSessionHandshake:476(2837ns) {SwitchToState:178(1201ns)}}}}}
2025-11-11T17:20:00.261298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518819256531254:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49195
2025-11-11T17:20:00.274372Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045367:2080] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.229969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518821423515167:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13426
2025-11-11T17:20:00.230261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518817128547848:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(147789255ns) {Configure:104(147784225ns) {ProcessConfigured:135(4181747ns) {StartInitialHandshake:157(4178978ns) {DropHandshakes:469(1437ns) {DropIncomingHandshake:441(339ns) DropOutgoingHandshake:456(269ns)} GenerateSessionVirtualId:988(379ns) PrepareNewSessionHandshake:476(1557ns) {SwitchToState:178(355ns)}}}}}
2025-11-11T17:20:00.277889Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515167:2064] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.294048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518817128547848:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.274097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518819256531254:2072] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.290022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045363:2079] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.298267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518816719939776:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(192378191ns) {Configure:104(192370057ns) {ProcessConfigured:135(61167ns) {StartInitialHandshake:157(58748ns) {DropHandshakes:469(3166ns) {DropIncomingHandshake:441(955ns) DropOutgoingHandshake:456(764ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:476(1817ns) {SwitchToState:178(571ns)}}}}}
2025-11-11T17:20:00.314884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518821423515171:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13426
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.282914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518814961563928:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(84654379ns) {Configure:104(84648957ns) {ProcessConfigured:135(72206ns) {StartInitialHandshake:157(69295ns) {DropHandshakes:469(3672ns) {DropIncomingHandshake:441(1103ns) DropOutgoingHandshake:456(684ns)} GenerateSessionVirtualId:988(1014ns) PrepareNewSessionHandshake:476(1839ns) {SwitchToState:178(715ns)}}}}}
2025-11-11T17:20:00.308543Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045356:2075] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.346369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907087:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.362435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518821423515177:2071] [node 5] ICP01 ready to work
2025-11-11T17:20:00.346629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907088:2061] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.347706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518819256531261:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49195
2025-11-11T17:20:00.308572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518821798045356:2075] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.538136Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518821423515167:2064] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:00.538383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518817128547848:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.538442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:00.538493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518817128547848:2048] [node 1] ICP22 created new session: [3:7571518821423515218:2048]
2025-11-11T17:20:00.538538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518821423515218:2048] [node 1] ICS09 handshake done sender: [3:7571518821423515167:2064] self: [3:7571518821423515170:0] peer: [1:7571518821798045355:0] socket: 382 qp: -1
2025-11-11T17:20:00.538578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518821423515218:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:00.538638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:00.493458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518821397968154:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.315155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518821798045373:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30587
2025-11-11T17:20:00.578279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518820493443099:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
2025-11-11T17:20:00.509726Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518821014907087:2060] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.653416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518820493443151:2068] [node 6] ICP01 ready to work
2025-11-11T17:20:00.647182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518817103000832:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(153460236ns) {Configure:104(153455527ns) {ProcessConfigured:135(77879ns) {StartInitialHandshake:157(74881ns) {DropHandshakes:469(3867ns) {DropIncomingHandshake:441(798ns) DropOutgoingHandshake:456(687ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:476(2237ns) {SwitchToState:178(713ns)}}}}}
2025-11-11T17:20:00.318029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518821798045356:2075] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:00.621661Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518821014907088:2061] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.709736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518821014907141:2066] [node 4] ICP01 ready to work
2025-11-11T17:20:00.710881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518814961563927:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(236908046ns) {Configure:104(236902799ns) {ProcessConfigured:135(110440ns) {StartInitialHandshake:157(107660ns) {DropHandshakes:469(3144ns) {DropIncomingHandshake:441(665ns) DropOutgoingHandshake:456(699ns)} GenerateSessionVirtualId:988(855ns) PrepareNewSessionHandshake:476(5140ns) {SwitchToState:178(693ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.333832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045373:2082] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.736690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518821397968154:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(89269475ns) {Configure:104(89261601ns) {ProcessConfigured:135(14656812ns) {StartInitialHandshake:157(14652922ns) {DropHandshakes:469(12493ns) {DropIncomingHandshake:441(9593ns) DropOutgoingHandshake:456(600ns)} GenerateSessionVirtualId:988(1206ns) PrepareNewSessionHandshake:476(4428ns) {SwitchToState:178(2343ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.713980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518819256531305:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49195
No RDMA context found for GID: 2025-11-11T17:20:00.738979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518821397968201:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:00.729934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518821423515226:2076] [node 4] ICP01 ready to work
2025-11-11T17:20:00.739141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518821397968202:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:00.739242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518821397968203:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:00.745796Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518821397968203:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-11T17:20:00.749648Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518820493443099:2065] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.727484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518816719939778:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17478653ns) {Configure:104(17471775ns) {ProcessConfigured:135(107262ns) {StartInitialHandshake:157(104101ns) {DropHandshakes:469(3348ns) {DropIncomingHandshake:441(776ns) DropOutgoingHandshake:456(603ns)} GenerateSessionVirtualId:988(1116ns) PrepareNewSessionHandshake:476(2751ns) {SwitchToState:178(984ns)}}}}}
2025-11-11T17:20:00.774056Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518819256531305:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.353669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518817503078029:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.727761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907089:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.729398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518821014907141:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.735808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518816719939776:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.736065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907144:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.756779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518821014907153:2073] [node 6] ICP01 ready to work
2025-11-11T17:20:00.757929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518821014907144:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.769073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518816719939778:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.769352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907156:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.797679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518821014907089:2062] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.802887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518819256531261:2074] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.803036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518814961563927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.804491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518819256531307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49195
2025-11-11T17:20:00.810466Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518819256531249:2069] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.810495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518819256531249:2069] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.813098Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518819256531261:2074] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.809922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518816719939777:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.813493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907159:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.373660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:00.777029Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518821397968202:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.817631Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518820493443096:2063] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.793953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518817103000831:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(324530473ns) {Configure:104(324524609ns) {ProcessConfigured:135(19967689ns) {StartInitialHandshake:157(19964774ns) {DropHandshakes:469(3158ns) {DropIncomingHandshake:441(834ns) DropOutgoingHandshake:456(657ns)} GenerateSessionVirtualId:988(586ns) PrepareNewSessionHandshake:476(5762ns) {SwitchToState:178(2477ns)}}}}}
2025-11-11T17:20:00.417677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518817503078029:2049] [node 3] ICP22 created new session: [1:7571518821798045378:2049]
2025-11-11T17:20:00.821860Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518819256531249:2069] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:00.830251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518814961563927:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.830318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:00.830380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518814961563927:2048] [node 1] ICP22 created new session: [2:7571518819256531310:2048]
2025-11-11T17:20:00.830427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518819256531310:2048] [node 1] ICS09 handshake done sender: [2:7571518819256531261:2074] self: [2:7571518819256531306:0] peer: [1:7571518821798045361:0] socket: 398 qp: -1
2025-11-11T17:20:00.830471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518819256531310:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:00.830508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:00.831205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518819256531253:2071] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.831230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518819256531253:2071] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.834343Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518819256531253:2071] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:00.834567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518819256531244:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.834603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:00.834645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518819256531244:2064] [node 5] ICP22 created new session: [2:7571518819256531314:2064]
2025-11-11T17:20:00.834687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518819256531314:2064] [node 5] ICS09 handshake done sender: [2:7571518819256531249:2069] self: [2:7571518819256531247:0] peer: [5:7571518821014907158:0] socket: 386 qp: -1
2025-11-11T17:20:00.834718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518819256531314:2064] [node 5] ICS10 traffic start
2025-11-11T17:20:00.834750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:00.835121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518819256531248:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.835168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:00.835218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518819256531248:2067] [node 4] ICP22 created new session: [2:7571518819256531316:2067]
2025-11-11T17:20:00.835255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518819256531316:2067] [node 4] ICS09 handshake done sender: [2:7571518819256531253:2071] self: [2:7571518819256531252:0] peer: [4:7571518820493443157:0] socket: 389 qp: -1
2025-11-11T17:20:00.835280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518819256531316:2067] [node 4] ICS10 traffic start
2025-11-11T17:20:00.835302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.813823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515179:2072] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.794131Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518821397968201:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.801680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518817103000830:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17493179ns)
2025-11-11T17:20:00.833904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518817103000830:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.841729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518817103000831:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.817685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518820493443096:2063] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.857680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518817103000832:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.813856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518821423515179:2072] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.509703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518821798045378:2049] [node 3] ICS09 handshake done sender: [1:7571518821798045356:2075] self: [1:7571518821798045355:0] peer: [3:7571518821423515170:0] socket: 379 qp: -1
2025-11-11T17:20:00.869838Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518819256531251:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.869876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518819256531251:2070] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.818815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518816198475781:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.883700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518821397968210:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:00.883809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518821397968211:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:00.884274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518821397968212:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:00.884342Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518821397968203:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:00.884507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518817103000831:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.884569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:00.884617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518817103000831:2049] [node 2] ICP22 created new session: [6:7571518821397968213:2049]
2025-11-11T17:20:00.884657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518821397968213:2049] [node 2] ICS09 handshake done sender: [6:7571518821397968203:2116] self: [6:7571518821397968208:0] peer: [2:7571518819256531250:0] socket: 407 qp: -1
2025-11-11T17:20:00.818917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518820493443154:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
2025-11-11T17:20:00.884704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518821397968213:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:00.884753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:00.879171Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:00.530724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518821798045378:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:00.577641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:00.754410Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045360:2077] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.754435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518821798045360:2077] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.797757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518821798045414:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30587
2025-11-11T17:20:00.801640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518817503078029:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(388163445ns) {HandleHandshakeStatus:329(388157550ns) {DropIncomingHandshake:441(20589ns) DropOutgoingHandshake:456(641ns) SwitchToState:178(1001ns) ProcessPendingSessionEvents:495(64992471ns) {ForwardSessionEventToSession:612(64980091ns) ForwardSessionEventToSession:612(5476ns)}}}
2025-11-11T17:20:00.803459Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045362:2078] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.803482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518821798045362:2078] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.810275Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518821798045362:2078] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:00.811646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518817503078028:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.811693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:00.811738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518817503078028:2048] [node 2] ICP22 created new session: [1:7571518821798045417:2048]
2025-11-11T17:20:00.811787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518821798045417:2048] [node 2] ICS09 handshake done sender: [1:7571518821798045362:2078] self: [1:7571518821798045361:0] peer: [2:7571518819256531306:0] socket: 397 qp: -1
2025-11-11T17:20:00.811817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518821798045417:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:00.811879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:00.827192Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518821798045360:2077] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:00.853827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045414:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.856448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518821798045354:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.856512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:00.856555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518821798045354:2069] [node 5] ICP22 created new session: [1:7571518821798045420:2069]
2025-11-11T17:20:00.856597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518821798045420:2069] [node 5] ICS09 handshake done sender: [1:7571518821798045360:2077] self: [1:7571518821798045359:0] peer: [5:7571518821014907150:0] socket: 381 qp: -1
2025-11-11T17:20:00.856633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518821798045420:2069] [node 5] ICS10 traffic start
2025-11-11T17:20:00.856674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:00.881900Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045358:2076] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.881931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518821798045358:2076] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.889995Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518821798045358:2076] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:00.902636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518821798045353:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.902715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:00.902764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518821798045353:2068] [node 6] ICP22 created new session: [1:7571518821798045424:2068]
2025-11-11T17:20:00.902813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518821798045424:2068] [node 6] ICS09 handshake done sender: [1:7571518821798045358:2076] self: [1:7571518821798045357:0] peer: [6:7571518821397968207:0] socket: 380 qp: -1
2025-11-11T17:20:00.902855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518821798045424:2068] [node 6] ICS10 traffic start
2025-11-11T17:20:00.902891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.846608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518821423515227:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13426
2025-11-11T17:20:00.914331Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.917984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518817103000830:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518821397968202:2115] held: no
2025-11-11T17:20:00.919539Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518821397968202:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:00.819013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518820493443155:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.846766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518821423515228:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13426
2025-11-11T17:20:00.882131Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518819256531251:2070] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:00.890387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518816719939776:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518821014907088:2061] held: no
2025-11-11T17:20:00.829733Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518820493443096:2063] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:00.901658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518819256531259:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.846895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518821423515231:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13426
2025-11-11T17:20:00.858481Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515163:2063] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.858504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518821423515163:2063] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.861687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515231:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.862021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518817128547849:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.881652Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518821423515179:2072] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:00.889783Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518821423515163:2063] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:00.937932Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515227:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.938030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515228:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.938198Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:00.939337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518821423515161:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.939400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:00.939449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518821423515161:2061] [node 6] ICP22 created new session: [3:7571518821423515236:2061]
2025-11-11T17:20:00.939490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518821423515236:2061] [node 6] ICS09 handshake done sender: [3:7571518821423515163:2063] self: [3:7571518821423515162:0] peer: [6:7571518821397968209:0] socket: 377 qp: -1
2025-11-11T17:20:00.939530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518821423515236:2061] [node 6] ICS10 traffic start
2025-11-11T17:20:00.939565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:00.940010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518821423515177:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.940056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:00.940089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518821423515177:2071] [node 5] ICP22 created new session: [3:7571518821423515238:2071]
2025-11-11T17:20:00.940124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518821423515238:2071] [node 5] ICS09 handshake done sender: [3:7571518821423515179:2072] self: [3:7571518821423515178:0] peer: [5:7571518821014907157:0] socket: 383 qp: -1
2025-11-11T17:20:00.940161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518821423515238:2071] [node 5] ICS10 traffic start
2025-11-11T17:20:00.940184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:00.940641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518817128547849:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518821423515231:2119] held: no
2025-11-11T17:20:00.940965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518821423515161:2061] [node 6] ICP18 handshake (actor [3:7571518821423515228:2117]) is too old
2025-11-11T17:20:00.941069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518821423515177:2071] [node 5] ICP18 handshake (actor [3:7571518821423515227:2116]) is too old
2025-11-11T17:20:00.941161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518821423515235:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13426
2025-11-11T17:20:00.945650Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518821397968201:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:00.959189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518817103000830:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.959257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:00.959322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518817103000830:2048] [node 1] ICP22 created new session: [6:7571518821397968216:2048]
2025-11-11T17:20:00.959367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518821397968216:2048] [node 1] ICS09 handshake done sender: [6:7571518821397968202:2115] self: [6:7571518821397968207:0] peer: [1:7571518821798045357:0] socket: 406 qp: -1
2025-11-11T17:20:00.959402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518821397968216:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:00.959467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:00.960011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518817103000832:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.960070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:00.960109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518817103000832:2050] [node 3] ICP22 created new session: [6:7571518821397968218:2050]
2025-11-11T17:20:00.960144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518821397968218:2050] [node 3] ICS09 handshake done sender: [6:7571518821397968201:2114] self: [6:7571518821397968209:0] peer: [3:7571518821423515162:0] socket: 405 qp: -1
2025-11-11T17:20:00.960173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518821397968218:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:00.960221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:00.901708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518819256531259:2073] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:00.920081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518819256531246:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.920123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:00.920167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518819256531246:2066] [node 6] ICP22 created new session: [2:7571518819256531319:2066]
2025-11-11T17:20:00.920205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518819256531319:2066] [node 6] ICS09 handshake done sender: [2:7571518819256531251:2070] self: [2:7571518819256531250:0] peer: [6:7571518821397968208:0] socket: 388 qp: -1
2025-11-11T17:20:00.920235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518819256531319:2066] [node 6] ICS10 traffic start
2025-11-11T17:20:00.920266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:00.921909Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518819256531259:2073] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:00.922153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518814961563928:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.922223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:00.922264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518814961563928:2049] [node 3] ICP22 created new session: [2:7571518819256531322:2049]
2025-11-11T17:20:00.922301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518819256531322:2049] [node 3] ICS09 handshake done sender: [2:7571518819256531259:2073] self: [2:7571518819256531258:0] peer: [3:7571518821423515233:0] socket: 422 qp: -1
2025-11-11T17:20:00.922333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518819256531322:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:00.922378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:00.854951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518820493443151:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(157089667ns) {Configure:104(157079107ns) {ProcessConfigured:135(76043ns) {StartInitialHandshake:157(73691ns) {DropHandshakes:469(5478ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(655ns)} GenerateSessionVirtualId:988(755ns) PrepareNewSessionHandshake:476(2258ns) {SwitchToState:178(662ns)}}}}}
2025-11-11T17:20:00.890760Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518821014907088:2061] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:00.905980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518816719939776:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.906060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:00.906106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518816719939776:2048] [node 1] ICP22 created new session: [5:7571518821014907163:2048]
2025-11-11T17:20:00.906158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518821014907163:2048] [node 1] ICS09 handshake done sender: [5:7571518821014907088:2061] self: [5:7571518821014907150:0] peer: [1:7571518821798045359:0] socket: 395 qp: -1
2025-11-11T17:20:00.906194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518821014907163:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:00.906240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:00.906669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907160:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.913868Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518821014907089:2062] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:00.930042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907161:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.930645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518816719939777:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.930721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:00.930764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518816719939777:2049] [node 2] ICP22 created new session: [5:7571518821014907165:2049]
2025-11-11T17:20:00.930805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518821014907165:2049] [node 2] ICS09 handshake done sender: [5:7571518821014907089:2062] self: [5:7571518821014907158:0] peer: [2:7571518819256531247:0] socket: 396 qp: -1
2025-11-11T17:20:00.930837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518821014907165:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:00.930868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:00.931545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518821014907162:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10832
2025-11-11T17:20:00.931961Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518821014907144:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:00.941554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518816719939778:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.941849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:00.945886Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518821014907087:2060] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:00.954478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518816719939778:2050] [node 3] ICP22 created new session: [5:7571518821014907168:2050]
2025-11-11T17:20:00.965738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518821014907168:2050] [node 3] ICS09 handshake done sender: [5:7571518821014907144:2115] self: [5:7571518821014907157:0] peer: [3:7571518821423515178:0] socket: 402 qp: -1
2025-11-11T17:20:00.965779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518821014907168:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:00.965815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:00.966212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518816719939778:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(24496367ns) {HandleHandshakeStatus:329(24494920ns) {DropIncomingHandshake:441(518ns) DropOutgoingHandshake:456(3454ns) SwitchToState:178(598ns) ProcessPendingSessionEvents:495(110521ns) {ForwardSessionEventToSession:612(59437ns) ForwardSessionEventToSession:612(5450ns) ForwardSessionEventToSession:612(2810ns) ForwardSessionEventToSession:612(5618ns) ForwardSessionEventToSession:612(2793ns) ForwardSessionEventToSession:612(3405ns) ForwardSessionEventToSession:612(4730ns) ForwardSessionEventToSession:612(2641ns) ForwardSessionEventToSession:612(8224ns) ForwardSessionEventToSession:612(7877ns)}}}
2025-11-11T17:20:00.967208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518821014907141:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:00.967279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:00.967323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518821014907141:2066] [node 4] ICP22 created new session: [5:7571518821014907171:2066]
2025-11-11T17:20:00.967364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518821014907171:2066] [node 4] ICS09 handshake done sender: [5:7571518821014907087:2060] self: [5:7571518821014907147:0] peer: [4:7571518820493443095:0] socket: 387 qp: -1
2025-11-11T17:20:00.967394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518821014907171:2066] [node 4] ICS10 traffic start
2025-11-11T17:20:00.967427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:00.968052Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518821014907162:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:01.002413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518825692935516:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:00.855281Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518820493443097:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.000039Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518821423515231:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:00.855363Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:00.916118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518816198475781:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518820493443099:2065] held: no
2025-11-11T17:20:00.916325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518820493443154:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.916406Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518820493443155:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:00.916475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518820493443091:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:00.916552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:00.916606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518820493443091:2061] [node 5] ICP22 created new session: [4:7571518820493443161:2061]
2025-11-11T17:20:00.916649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518820493443161:2061] [node 5] ICS09 handshake done sender: [4:7571518820493443096:2063] self: [4:7571518820493443095:0] peer: [5:7571518821014907147:0] socket: 376 qp: -1
2025-11-11T17:20:00.916685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518820493443161:2061] [node 5] ICS10 traffic start
2025-11-11T17:20:00.916757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:00.917415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518820493443160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
2025-11-11T17:20:00.969948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518820493443091:2061] [node 5] ICP18 handshake (actor [4:7571518820493443154:2116]) is too old
2025-11-11T17:20:00.970119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518816198475780:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:00.993698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518820493443151:2068] [node 6] ICP35 event processing took too much time PendingConnection:278(28253370ns) {IncomingHandshake:261(12152203ns) {DropIncomingHandshake:441(867ns) PrepareNewSessionHandshake:476(7983154ns) {SwitchToState:178(3884ns)} GenerateSessionVirtualId:988(1025ns) IssueIncomingHandshakeReply:193(8069ns)}}
2025-11-11T17:20:01.001704Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518820493443099:2065] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:01.010339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518824788410463:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
2025-11-11T17:20:01.029790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518817128547849:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:01.029875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:01.029961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518817128547849:2049] [node 2] ICP22 created new session: [3:7571518825718482538:2049]
2025-11-11T17:20:01.030002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518825718482538:2049] [node 2] ICS09 handshake done sender: [3:7571518821423515231:2119] self: [3:7571518821423515233:0] peer: [2:7571518819256531258:0] socket: 384 qp: -1
2025-11-11T17:20:01.030042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518825718482538:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:01.030081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:01.002496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518825692935517:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:01.033744Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518821798045369:2081] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.033798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518821798045369:2081] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:01.038915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518825692935517:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.039036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518821397968139:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:01.025766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518816198475781:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:01.057935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518821423515241:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13426
2025-11-11T17:20:01.062032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518825692935519:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:01.062652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515230:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.025889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:01.062697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518821423515230:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:01.079739Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518821423515230:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:01.090070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518821423515226:2076] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:01.090147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:01.090200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518821423515226:2076] [node 4] ICP22 created new session: [3:7571518825718482541:2076]
2025-11-11T17:20:01.090248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518825718482541:2076] [node 4] ICS09 handshake done sender: [3:7571518821423515230:2118] self: [3:7571518821423515229:0] peer: [4:7571518824788410466:0] socket: 404 qp: -1
2025-11-11T17:20:01.090286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518825718482541:2076] [node 4] ICS10 traffic start
2025-11-11T17:20:01.090324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:01.079195Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518825692935516:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:01.079300Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518825692935517:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:01.092277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518821397968139:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:01.092327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:01.092369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518821397968139:2059] [node 4] ICP22 created new session: [6:7571518825692935520:2059]
2025-11-11T17:20:01.092409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518825692935520:2059] [node 4] ICS09 handshake done sender: [6:7571518825692935517:2122] self: [6:7571518825692935518:0] peer: [4:7571518820493443158:0] socket: 410 qp: -1
2025-11-11T17:20:01.092439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518825692935520:2059] [node 4] ICS10 traffic start
2025-11-11T17:20:01.092499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:01.092877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518821397968154:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:01.025936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518816198475781:2049] [node 2] ICP22 created new session: [4:7571518824788410464:2049]
2025-11-11T17:20:01.026003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518824788410464:2049] [node 2] ICS09 handshake done sender: [4:7571518820493443099:2065] self: [4:7571518820493443157:0] peer: [2:7571518819256531252:0] socket: 390 qp: -1
2025-11-11T17:20:01.026051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518824788410464:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:01.026090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:01.038268Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518824788410463:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.038421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518816198475782:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:01.045869Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518820493443159:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.045898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518820493443159:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:01.053844Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518820493443159:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:01.093876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518820493443151:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:01.094747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:01.094795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518820493443151:2068] [node 6] ICP22 created new session: [4:7571518824788410470:2068]
2025-11-11T17:20:01.094842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518824788410470:2068] [node 6] ICS09 handshake done sender: [4:7571518820493443159:2118] self: [4:7571518820493443158:0] peer: [6:7571518825692935518:0] socket: 401 qp: -1
2025-11-11T17:20:01.094868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518824788410470:2068] [node 6] ICS10 traffic start
2025-11-11T17:20:01.094915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:01.095746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518824788410468:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
2025-11-11T17:20:01.094023Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518821423515241:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.094139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518821423515226:2076] [node 4] ICP18 handshake (actor [3:7571518821423515241:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:01.097329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518824788410469:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29026
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:01.098792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518825692935523:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11846
2025-11-11T17:20:01.097973Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:01.097779Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518821014907155:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:01.097816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518821014907155:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:01.097818Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518821798045369:2081] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:01.099131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518821798045352:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:01.099181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:01.099220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518821798045352:2067] [node 4] ICP22 created new session: [1:7571518826093012723:2067]
2025-11-11T17:20:01.099281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518826093012723:2067] [node 4] ICS09 handshake done sender: [1:7571518821798045369:2081] self: [1:7571518821798045368:0] peer: [4:7571518820493443166:0] socket: 415 qp: -1
2025-11-11T17:20:01.099316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518826093012723:2067] [node 4] ICS10 traffic start
2025-11-11T17:20:01.099344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:01.110050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518816198475782:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518824788410463:2120] held: no
2025-11-11T17:20:01.110257Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:01.111306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518816198475780:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518820493443155:2117] held: no
2025-11-11T17:20:01.099434Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518821014907155:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:01.099927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518821014907153:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:01.099998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:01.100039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518821014907153:2073] [node 6] ICP22 created new session: [5:7571518825309874471:2073]
2025-11-11T17:20:01.100073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518825309874471:2073] [node 6] ICS09 handshake done sender: [5:7571518821014907155:2118] self: [5:7571518821014907154:0] peer: [6:7571518825692935522:0] socket: 373 qp: -1
2025-11-11T17:20:01.100096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518825309874471:2073] [node 6] ICS10 traffic start
2025-11-11T17:20:01.100132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:01.118008Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20522) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:01.118972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518821397968154:2069] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20522) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518825692935516:2121] held: no
2025-11-11T17:20:01.119093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518825692935516:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:01.137858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518821397968154:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:01.137968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:01.138022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518821397968154:2069] [node 5] ICP22 created new session: [6:7571518825692935524:2069]
2025-11-11T17:20:01.138064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518825692935524:2069] [node 5] ICS09 handshake done sender: [6:7571518825692935516:2121] self: [6:7571518825692935522:0] peer: [5:7571518821014907154:0] socket: 403 qp: -1
2025-11-11T17:20:01.138113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518825692935524:2069] [node 5] ICS10 traffic start
2025-11-11T17:20:01.138175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:01.137858Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518824788410463:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:01.138779Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518820493443155:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:01.141977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518816198475782:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:01.142066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:01.142114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518816198475782:2050] [node 3] ICP22 created new session: [4:7571518824788410472:2050]
2025-11-11T17:20:01.142157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518824788410472:2050] [node 3] ICS09 handshake done sender: [4:7571518824788410463:2120] self: [4:7571518824788410466:0] peer: [3:7571518821423515229:0] socket: 412 qp: -1
2025-11-11T17:20:01.142193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518824788410472:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:01.142233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:01.149788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518816198475780:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:01.149880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:01.149929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518816198475780:2048] [node 1] ICP22 created new session: [4:7571518824788410474:2048]
2025-11-11T17:20:01.149979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518824788410474:2048] [node 1] ICS09 handshake done sender: [4:7571518820493443155:2117] self: [4:7571518820493443166:0] peer: [1:7571518821798045368:0] socket: 419 qp: -1
2025-11-11T17:20:01.150015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518824788410474:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:01.150047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:01.195061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:01.195308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518819256531310:2048] [node 1] ICS01 socket: 398 reason# EndOfStream
2025-11-11T17:20:01.195349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518814961563927:2048] [node 1] ICP30 unregister session Session# [2:7571518819256531310:2048] VirtualId# [2:7571518819256531306:0]
2025-11-11T17:20:01.195382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518819256531310:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.195465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:01.195505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:01.197325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:01.197535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518821423515218:2048] [node 1] ICS01 socket: 382 reason# EndOfStream
2025-11-11T17:20:01.197565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518817128547848:2048] [node 1] ICP30 unregister session Session# [3:7571518821423515218:2048] VirtualId# [3:7571518821423515170:0]
2025-11-11T17:20:01.197621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518821423515218:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.197688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:01.197720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:01.205780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:01.206025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518824788410474:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-11T17:20:01.206062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518816198475780:2048] [node 1] ICP30 unregister session Session# [4:7571518824788410474:2048] VirtualId# [4:7571518820493443166:0]
2025-11-11T17:20:01.206099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518824788410474:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.206179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:01.206222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:01.206313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:01.206525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518821397968216:2048] [node 1] ICS01 socket: 406 reason# EndOfStream
2025-11-11T17:20:01.206555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518817103000830:2048] [node 1] ICP30 unregister session Session# [6:7571518821397968216:2048] VirtualId# [6:7571518821397968207:0]
2025-11-11T17:20:01.206589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518821397968216:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.206642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:01.206673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:01.206743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:01.206914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518821014907163:2048] [node 1] ICS01 socket: 395 reason# EndOfStream
2025-11-11T17:20:01.206944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518816719939776:2048] [node 1] ICP30 unregister session Session# [5:7571518821014907163:2048] VirtualId# [5:7571518821014907150:0]
2025-11-11T17:20:01.206965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518821014907163:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.207013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:01.207042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:01.265710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:01.265990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518825718482538:2049] [node 2] ICS01 socket: 384 reason# EndOfStream
2025-11-11T17:20:01.266052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518817128547849:2049] [node 2] ICP30 unregister session Session# [3:7571518825718482538:2049] VirtualId# [3:7571518821423515233:0]
2025-11-11T17:20:01.266088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518825718482538:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.266217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:01.266266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:01.274459Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:01.274640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518821397968213:2049] [node 2] ICS01 socket: 407 reason# EndOfStream
2025-11-11T17:20:01.274677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518817103000831:2049] [node 2] ICP30 unregister session Session# [6:7571518821397968213:2049] VirtualId# [6:7571518821397968208:0]
2025-11-11T17:20:01.274700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518821397968213:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.274756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:01.274789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:01.274869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:01.275104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518824788410464:2049] [node 2] ICS01 socket: 390 reason# EndOfStream
2025-11-11T17:20:01.275132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518816198475781:2049] [node 2] ICP30 unregister session Session# [4:7571518824788410464:2049] VirtualId# [4:7571518820493443157:0]
2025-11-11T17:20:01.275179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518824788410464:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.275239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:01.275274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:01.273922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:01.274166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518821014907165:2049] [node 2] ICS01 socket: 396 reason# EndOfStream
2025-11-11T17:20:01.274209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518816719939777:2049] [node 2] ICP30 unregister session Session# [5:7571518821014907165:2049] VirtualId# [5:7571518821014907158:0]
2025-11-11T17:20:01.274242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518821014907165:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.274330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:01.274378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:01.421808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:01.422045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518821397968218:2050] [node 3] ICS01 socket: 405 reason# EndOfStream
2025-11-11T17:20:01.422084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518817103000832:2050] [node 3] ICP30 unregister session Session# [6:7571518821397968218:2050] VirtualId# [6:7571518821397968209:0]
2025-11-11T17:20:01.422115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518821397968218:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.422223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:01.422275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:01.422363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:01.422564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518821014907168:2050] [node 3] ICS01 socket: 402 reason# EndOfStream
2025-11-11T17:20:01.422594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518816719939778:2050] [node 3] ICP30 unregister session Session# [5:7571518821014907168:2050] VirtualId# [5:7571518821014907157:0]
2025-11-11T17:20:01.422625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518821014907168:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.422702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:01.422755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:01.425716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:01.425942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518824788410472:2050] [node 3] ICS01 socket: 412 reason# EndOfStream
2025-11-11T17:20:01.425975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518816198475782:2050] [node 3] ICP30 unregister session Session# [4:7571518824788410472:2050] VirtualId# [4:7571518824788410466:0]
2025-11-11T17:20:01.426000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518824788410472:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.426087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:01.426127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:01.529868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:01.530205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518821014907171:2066] [node 4] ICS01 socket: 387 reason# EndOfStream
2025-11-11T17:20:01.530251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518821014907141:2066] [node 4] ICP30 unregister session Session# [5:7571518821014907171:2066] VirtualId# [5:7571518821014907147:0]
2025-11-11T17:20:01.530285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518821014907171:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.530424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:01.530474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:01.537728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:01.538113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518825692935520:2059] [node 4] ICS01 socket: 410 reason# EndOfStream
2025-11-11T17:20:01.538188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518821397968139:2059] [node 4] ICP30 unregister session Session# [6:7571518825692935520:2059] VirtualId# [6:7571518825692935518:0]
2025-11-11T17:20:01.538250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518825692935520:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.538438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:01.538523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:01.661755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:01.662020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518825692935524:2069] [node 5] ICS01 socket: 403 reason# EndOfStream
2025-11-11T17:20:01.662062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518821397968154:2069] [node 5] ICP30 unregister session Session# [6:7571518825692935524:2069] VirtualId# [6:7571518825692935522:0]
2025-11-11T17:20:01.662096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518825692935524:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:01.662201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:01.662243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:01.882496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518823273978226:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:01.882541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518823273978227:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:01.957905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518824742957365:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:01.957951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518824742957366:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:02.021912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518823147718666:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:02.021967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518823147718667:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:02.097881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518828439904623:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:02.097935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518828439904624:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:02.097963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518828439904625:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:02.145900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518830645013517:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:02.145943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518830645013518:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:02.145968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518830645013519:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:02.186085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518828512436357:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:02.186133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518828512436358:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:02.186160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518828512436359:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:02.494555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518827568945582:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:02.490824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518830645013535:2064] [node 4] ICP01 ready to work
2025-11-11T17:20:02.550569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518827442686024:2106] [node 5] ICP01 ready to work
2025-11-11T17:20:02.567868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518828512436375:2063] [node 4] ICP01 ready to work
2025-11-11T17:20:02.585692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518827442686033:2097] [node 6] ICP01 ready to work
2025-11-11T17:20:02.593680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518828439904671:2063] [node 5] ICP01 ready to work
2025-11-11T17:20:02.600651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518830645013535:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(109475376ns) {Configure:104(109469720ns) {ProcessConfigured:135(77069ns) {StartInitialHandshake:157(73477ns) {DropHandshakes:469(3352ns) {DropIncomingHandshake:441(870ns) DropOutgoingHandshake:456(582ns)} GenerateSessionVirtualId:988(1569ns) PrepareNewSessionHandshake:476(1967ns) {SwitchToState:178(495ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.617185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518827568945586:2107] [node 5] ICP01 ready to work
2025-11-11T17:20:02.653743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518827568945587:2109] [node 4] ICP01 ready to work
2025-11-11T17:20:02.655360Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518827568945582:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(160511878ns) {Configure:104(160506756ns) {ProcessConfigured:135(67672ns) {StartInitialHandshake:157(64872ns) {DropHandshakes:469(8800ns) {DropIncomingHandshake:441(797ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(920ns) PrepareNewSessionHandshake:476(2060ns) {SwitchToState:178(699ns)}}}}}
2025-11-11T17:20:02.672144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518827568945592:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.712086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518830645013541:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:02.632280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518829037924724:2102] [node 6] ICP01 ready to work
2025-11-11T17:20:02.745657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518828439904676:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:02.653649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518827442686036:2095] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.730989Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945592:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.733930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518830645013579:2065] [node 6] ICP01 ready to work
2025-11-11T17:20:02.761725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518829037924725:2104] [node 4] ICP01 ready to work
2025-11-11T17:20:02.738447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518828512436359:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(245867403ns) {Configure:104(245852088ns) {ProcessConfigured:135(77141117ns) {StartInitialHandshake:157(77138261ns) {DropHandshakes:469(2532ns) {DropIncomingHandshake:441(577ns) DropOutgoingHandshake:456(284ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:476(3954ns) {SwitchToState:178(2145ns)}}}}}
2025-11-11T17:20:02.715653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518827442686037:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-11T17:20:02.777409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518827568945600:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:20:02.819168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518828512436375:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(237274500ns) {Configure:104(237262029ns) {ProcessConfigured:135(75655ns) {StartInitialHandshake:157(71720ns) {DropHandshakes:469(3572ns) {DropIncomingHandshake:441(613ns) DropOutgoingHandshake:456(776ns)} GenerateSessionVirtualId:988(1158ns) PrepareNewSessionHandshake:476(2493ns) {SwitchToState:178(776ns)}}}}}
2025-11-11T17:20:02.762517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518827442686038:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-11T17:20:02.762571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518827442686039:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-11T17:20:02.837382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518827442686036:2095] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(103433602ns) {Configure:104(103426751ns) {ProcessConfigured:135(77494ns) {StartInitialHandshake:157(74296ns) {DropHandshakes:469(3289ns) {DropIncomingHandshake:441(723ns) DropOutgoingHandshake:456(662ns)} GenerateSessionVirtualId:988(1484ns) PrepareNewSessionHandshake:476(2117ns) {SwitchToState:178(676ns)}}}}}
2025-11-11T17:20:02.761727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518829037924726:2106] [node 5] ICP01 ready to work
2025-11-11T17:20:02.798124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518830645013541:2069] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.847289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518828439904625:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(229439906ns) {Configure:104(229432354ns) {ProcessConfigured:135(59445ns) {StartInitialHandshake:157(57217ns) {DropHandshakes:469(3773ns) {DropIncomingHandshake:441(646ns) DropOutgoingHandshake:456(526ns)} GenerateSessionVirtualId:988(1061ns) PrepareNewSessionHandshake:476(2047ns) {SwitchToState:178(863ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.862825Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518827442686026:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.819914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518828512436429:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:02.883515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518828439904671:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(105474801ns) {Configure:104(105469270ns) {ProcessConfigured:135(75493ns) {StartInitialHandshake:157(72355ns) {DropHandshakes:469(3062ns) {DropIncomingHandshake:441(787ns) DropOutgoingHandshake:456(537ns)} GenerateSessionVirtualId:988(1031ns) PrepareNewSessionHandshake:476(2506ns) {SwitchToState:178(1003ns)}}}}}
2025-11-11T17:20:02.884040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518828439904690:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:02.884250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518828439904691:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:02.884375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518828439904692:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:02.869786Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945591:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.829345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518830645013519:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:02.874112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518824742957365:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(380164601ns) {Configure:104(380151477ns) {ProcessConfigured:135(62829ns) {StartInitialHandshake:157(57561ns) {DropHandshakes:469(3244ns) {DropIncomingHandshake:441(847ns) DropOutgoingHandshake:456(570ns)} GenerateSessionVirtualId:988(721ns) PrepareNewSessionHandshake:476(2774ns) {SwitchToState:178(1137ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.820115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518828512436430:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:02.862886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518827442686026:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:02.869827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518827568945591:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:02.885869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518823273978226:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(228741312ns) {Configure:104(228737165ns) {ProcessConfigured:135(27926914ns) {StartInitialHandshake:157(27923416ns) {DropHandshakes:469(1805ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(319ns)} GenerateSessionVirtualId:988(633ns) PrepareNewSessionHandshake:476(3888ns) {SwitchToState:178(1389ns)}}}}}
2025-11-11T17:20:02.889804Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518827568945591:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:02.909632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945589:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.909680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518827568945589:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.918732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518829037924727:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37474
2025-11-11T17:20:02.865779Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518827442686037:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.837978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518830645013517:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(125659671ns) {Configure:104(125655038ns) {ProcessConfigured:135(15801344ns) {StartInitialHandshake:157(15798592ns) {DropHandshakes:469(2564ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(573ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:476(3941ns) {SwitchToState:178(1441ns)}}}}}
2025-11-11T17:20:02.897802Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518828439904690:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.825853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518828512436406:2066] [node 5] ICP01 ready to work
2025-11-11T17:20:02.919026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518829037924734:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37474
2025-11-11T17:20:02.921986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518829037924724:2102] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17247778ns) {Configure:104(17243527ns) {ProcessConfigured:135(15317609ns) {StartInitialHandshake:157(15315044ns) {DropHandshakes:469(15251293ns) {DropIncomingHandshake:441(1834ns) DropOutgoingHandshake:456(574ns)} GenerateSessionVirtualId:988(2471ns) PrepareNewSessionHandshake:476(2804ns) {SwitchToState:178(1297ns)}}}}}
2025-11-11T17:20:02.929774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518829037924734:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.866706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518830645013518:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(265764747ns) {Configure:104(265760469ns) {ProcessConfigured:135(8817006ns) {StartInitialHandshake:157(8813021ns) {DropHandshakes:469(3249ns) {DropIncomingHandshake:441(876ns) DropOutgoingHandshake:456(557ns)} GenerateSessionVirtualId:988(1058ns) PrepareNewSessionHandshake:476(2748ns) {SwitchToState:178(165ns)}}}}}
2025-11-11T17:20:02.867075Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518828512436430:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.897966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518828439904691:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.970472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518827568945606:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:20:02.865937Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518827442686038:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.886755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518830645013597:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:02.979737Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518829037924727:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.866044Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518827442686039:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.867157Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518828512436429:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.922609Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518828439904692:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.978111Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518827568945589:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:03.021791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518829037924725:2104] [node 4] ICP35 event processing took too much time PendingConnection:278(26318818ns) {IncomingHandshake:261(9953487ns) {DropIncomingHandshake:441(1570ns) PrepareNewSessionHandshake:476(4191ns) {SwitchToState:178(3155ns)} GenerateSessionVirtualId:988(434ns) IssueIncomingHandshakeReply:193(9811ns)}}
2025-11-11T17:20:02.923119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518828439904701:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:02.866223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518823147718666:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:02.867306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518828512436406:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:02.893729Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518830645013582:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.970830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518828439904676:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(121354970ns) {Configure:104(121351047ns) {ProcessConfigured:135(4444481ns) {StartInitialHandshake:157(4441499ns) {DropHandshakes:469(2109ns) {DropIncomingHandshake:441(394ns) DropOutgoingHandshake:456(614ns)} GenerateSessionVirtualId:988(462ns) PrepareNewSessionHandshake:476(9432ns) {SwitchToState:178(191ns)}}}}}
2025-11-11T17:20:02.989895Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945600:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.981300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518828439904623:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(349243866ns) {Configure:104(349235220ns) {ProcessConfigured:135(14463766ns) {StartInitialHandshake:157(14460707ns) {DropHandshakes:469(1392ns) {DropIncomingHandshake:441(215ns) DropOutgoingHandshake:456(192ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:476(3659ns) {SwitchToState:178(635ns)}}}}}
2025-11-11T17:20:02.981444Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518828439904696:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.981466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518828439904696:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:02.991876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518828439904623:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.002406Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518828439904696:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:02.885401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518827442686044:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-11T17:20:02.893774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518830645013582:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:02.867358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518828512436357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.085785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518828439904701:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.990386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518823273978227:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:02.990465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:02.990508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518823273978227:2049] [node 3] ICP22 created new session: [1:7571518827568945607:2049]
2025-11-11T17:20:02.990552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518827568945607:2049] [node 3] ICS09 handshake done sender: [1:7571518827568945591:2111] self: [1:7571518827568945590:0] peer: [3:7571518827442686042:0] socket: 384 qp: -1
2025-11-11T17:20:02.990590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518827568945607:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:02.990634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:03.022098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945606:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.029685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518827568945582:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.029734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:03.029769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518827568945582:2108] [node 6] ICP22 created new session: [1:7571518831863912906:2108]
2025-11-11T17:20:03.029809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518831863912906:2108] [node 6] ICS09 handshake done sender: [1:7571518827568945589:2110] self: [1:7571518827568945588:0] peer: [6:7571518828512436436:0] socket: 383 qp: -1
2025-11-11T17:20:03.029836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518831863912906:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:03.029870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:03.050158Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945594:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.050187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518827568945594:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.082848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518827568945594:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:03.093680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518827568945582:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(16005685ns) {HandleHandshakeStatus:329(16004150ns) {DropIncomingHandshake:441(9545ns) DropOutgoingHandshake:456(255ns) SwitchToState:178(119ns) ProcessPendingSessionEvents:495(15700648ns) {ForwardSessionEventToSession:612(42329ns) ForwardSessionEventToSession:612(3119ns) ForwardSessionEventToSession:612(5155ns) ForwardSessionEventToSession:612(2595ns) ForwardSessionEventToSession:612(2491ns) ForwardSessionEventToSession:612(5630ns) ForwardSessionEventToSession:612(371511ns) ForwardSessionEventToSession:612(2741ns) ForwardSessionEventToSession:612(15242192ns) ForwardSessionEventToSession:612(5218ns)}}}
2025-11-11T17:20:02.958179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518830645013598:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:02.958232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518830645013599:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:02.961943Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518830645013597:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.965885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518830645013535:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.026402Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518830645013582:2107] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:03.026680Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518830645013541:2069] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:03.026879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518830645013519:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.026950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:03.026998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518830645013519:2050] [node 3] ICP22 created new session: [5:7571518834939980900:2050]
2025-11-11T17:20:03.027044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518834939980900:2050] [node 3] ICS09 handshake done sender: [5:7571518830645013541:2069] self: [5:7571518830645013593:0] peer: [3:7571518827442686025:0] socket: 381 qp: -1
2025-11-11T17:20:03.027074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518834939980900:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:03.027124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:03.027528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518834939980898:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:03.027839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518834939980899:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:03.028646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518830645013579:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.028710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:03.028747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518830645013579:2065] [node 6] ICP22 created new session: [5:7571518834939980902:2065]
2025-11-11T17:20:03.028788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518834939980902:2065] [node 6] ICS09 handshake done sender: [5:7571518830645013582:2107] self: [5:7571518830645013581:0] peer: [6:7571518828512436435:0] socket: 389 qp: -1
2025-11-11T17:20:03.028836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518834939980902:2065] [node 6] ICS10 traffic start
2025-11-11T17:20:03.028870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:03.034213Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518830645013597:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:03.074036Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518830645013599:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.074459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518830645013535:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.074508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:03.074554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518830645013535:2064] [node 4] ICP22 created new session: [5:7571518834939980905:2064]
2025-11-11T17:20:03.074600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518834939980905:2064] [node 4] ICS09 handshake done sender: [5:7571518830645013597:2116] self: [5:7571518830645013600:0] peer: [4:7571518828439904695:0] socket: 408 qp: -1
2025-11-11T17:20:03.074633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518834939980905:2064] [node 4] ICS10 traffic start
2025-11-11T17:20:03.074670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.109738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518828439904671:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.110399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518833332892039:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37474
2025-11-11T17:20:03.110466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518833332892040:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37474
2025-11-11T17:20:03.110524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518833332892041:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37474
2025-11-11T17:20:02.954044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518828512436439:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:02.885486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518827442686045:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.125815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518824742957366:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(236342484ns) {Configure:104(236331567ns) {ProcessConfigured:135(23220476ns) {StartInitialHandshake:157(16907529ns) {DropHandshakes:469(3254ns) {DropIncomingHandshake:441(1055ns) DropOutgoingHandshake:456(778ns)} GenerateSessionVirtualId:988(1900ns) PrepareNewSessionHandshake:476(3247ns) {SwitchToState:178(1062ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.914771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518827442686033:2097] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(264191052ns) {Configure:104(253730805ns) {ProcessConfigured:135(70237ns) {StartInitialHandshake:157(67143ns) {DropHandshakes:469(2957ns) {DropIncomingHandshake:441(777ns) DropOutgoingHandshake:456(487ns)} GenerateSessionVirtualId:988(1112ns) PrepareNewSessionHandshake:476(1790ns) {SwitchToState:178(840ns)}}}}}
2025-11-11T17:20:03.106036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518830645013579:2065] [node 6] ICP35 event processing took too much time PendingConnection:278(34890125ns) {HandleHandshakeStatus:329(34885850ns) {DropIncomingHandshake:441(496ns) DropOutgoingHandshake:456(916ns) SwitchToState:178(267ns) ProcessPendingSessionEvents:495(34555775ns) {ForwardSessionEventToSession:612(37298ns) ForwardSessionEventToSession:612(2970ns) ForwardSessionEventToSession:612(24693761ns) ForwardSessionEventToSession:612(22195ns) ForwardSessionEventToSession:612(3835ns) ForwardSessionEventToSession:612(4631329ns) ForwardSessionEventToSession:612(3509ns) ForwardSessionEventToSession:612(5128428ns) ForwardSessionEventToSession:612(7636ns) ForwardSessionEventToSession:612(3337ns)}}}
2025-11-11T17:20:03.113905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518827568945587:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:02.977812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518828512436358:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(172647229ns) {Configure:104(172640248ns) {ProcessConfigured:135(98115ns) {StartInitialHandshake:157(95477ns) {DropHandshakes:469(3174ns) {DropIncomingHandshake:441(591ns) DropOutgoingHandshake:456(544ns)} GenerateSessionVirtualId:988(1053ns) PrepareNewSessionHandshake:476(1938ns) {SwitchToState:178(678ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.973680Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518827442686026:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:03.113993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:03.142373Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518833332892040:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.150475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:02.993724Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518828512436429:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:03.110889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518830645013518:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.005727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518827442686045:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.142491Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518829037924733:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.150561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518828439904671:2063] [node 5] ICP22 created new session: [4:7571518832734872007:2063]
2025-11-11T17:20:03.114044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518827568945587:2109] [node 4] ICP22 created new session: [1:7571518831863912913:2109]
2025-11-11T17:20:03.111455Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.142511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518829037924733:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.143343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518824742957365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.143944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518829037924733:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:03.144249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518829037924726:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.144311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:03.144351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518829037924726:2106] [node 5] ICP22 created new session: [2:7571518833332892045:2106]
2025-11-11T17:20:03.144401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518833332892045:2106] [node 5] ICS09 handshake done sender: [2:7571518829037924733:2113] self: [2:7571518829037924732:0] peer: [5:7571518834939980909:0] socket: 413 qp: -1
2025-11-11T17:20:03.144435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518833332892045:2106] [node 5] ICS10 traffic start
2025-11-11T17:20:03.144469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:03.174288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518833332892048:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37474
2025-11-11T17:20:03.177649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518833332892039:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.190548Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518829037924731:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.190573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518829037924731:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.191071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518833332892040:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.994357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518828512436357:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.126213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518830645013579:2065] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:03.017689Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518827442686037:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:03.165705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518832734872007:2063] [node 5] ICS09 handshake done sender: [4:7571518828439904696:2115] self: [4:7571518828439904695:0] peer: [5:7571518830645013600:0] socket: 406 qp: -1
2025-11-11T17:20:03.173701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518832734872007:2063] [node 5] ICS10 traffic start
2025-11-11T17:20:03.178172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518828439904624:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.181825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:03.198296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518828439904671:2063] [node 5] ICP35 event processing took too much time PendingConnection:278(103926108ns) {HandleHandshakeStatus:329(103923658ns) {DropIncomingHandshake:441(20902ns) DropOutgoingHandshake:456(622ns) SwitchToState:178(737ns) ProcessPendingSessionEvents:495(117471ns) {ForwardSessionEventToSession:612(72593ns) ForwardSessionEventToSession:612(2945ns) ForwardSessionEventToSession:612(3049ns) ForwardSessionEventToSession:612(5530ns) ForwardSessionEventToSession:612(2982ns) ForwardSessionEventToSession:612(5482ns) ForwardSessionEventToSession:612(5684ns) ForwardSessionEventToSession:612(2022ns) ForwardSessionEventToSession:612(4119ns) ForwardSessionEventToSession:612(2054ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.229653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518824742957365:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.236856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:02.994410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:03.114116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518831863912913:2109] [node 4] ICS09 handshake done sender: [1:7571518827568945594:2113] self: [1:7571518827568945593:0] peer: [4:7571518828439904706:0] socket: 385 qp: -1
2025-11-11T17:20:03.217904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518832734872005:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:03.229743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518832734872006:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:03.238147Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518828439904690:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.034342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518827442686024:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:02.994477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518828512436357:2048] [node 1] ICP22 created new session: [6:7571518828512436440:2048]
2025-11-11T17:20:03.114156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518831863912913:2109] [node 4] ICS10 traffic start
2025-11-11T17:20:03.034432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:03.034483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518827442686024:2106] [node 5] ICP22 created new session: [3:7571518831737653346:2106]
2025-11-11T17:20:03.034525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518831737653346:2106] [node 5] ICS09 handshake done sender: [3:7571518827442686026:2110] self: [3:7571518827442686025:0] peer: [5:7571518830645013593:0] socket: 379 qp: -1
2025-11-11T17:20:03.034560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518831737653346:2106] [node 5] ICS10 traffic start
2025-11-11T17:20:03.034590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:03.035200Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:03.062426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518823147718666:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518827442686037:2113] held: no
2025-11-11T17:20:03.062533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518823147718666:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.062576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:03.062644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518823147718666:2048] [node 1] ICP22 created new session: [3:7571518831737653348:2048]
2025-11-11T17:20:03.062693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518831737653348:2048] [node 1] ICS09 handshake done sender: [3:7571518827442686037:2113] self: [3:7571518827442686042:0] peer: [1:7571518827568945590:0] socket: 386 qp: -1
2025-11-11T17:20:03.062735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518831737653348:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:03.062773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:03.214245Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518827442686047:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.214291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518827442686047:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.243691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518824742957365:2048] [node 1] ICP22 created new session: [2:7571518833332892050:2048]
2025-11-11T17:20:03.247950Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518832734872006:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:02.994521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518828512436440:2048] [node 1] ICS09 handshake done sender: [6:7571518828512436429:2113] self: [6:7571518828512436436:0] peer: [1:7571518827568945588:0] socket: 396 qp: -1
2025-11-11T17:20:03.142168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518834939980908:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:03.294664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518827442686041:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.114196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.248014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518828439904623:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.173822Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518834939980908:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.246911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518829037924726:2106] [node 5] ICP18 handshake (actor [2:7571518833332892039:2117]) is too old
2025-11-11T17:20:03.294711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518827442686041:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:02.994555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518828512436440:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:03.116056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518831863912912:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:20:03.326220Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518827442686047:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.247066Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:02.994593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:03.248070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:03.189821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518830645013517:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.061629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518832807403739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:03.066136Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518828512436430:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:03.066331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518828512436406:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.066381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:03.066425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518828512436406:2066] [node 5] ICP22 created new session: [6:7571518832807403740:2066]
2025-11-11T17:20:03.066476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518832807403740:2066] [node 5] ICS09 handshake done sender: [6:7571518828512436430:2114] self: [6:7571518828512436435:0] peer: [5:7571518830645013581:0] socket: 397 qp: -1
2025-11-11T17:20:03.066514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518832807403740:2066] [node 5] ICS10 traffic start
2025-11-11T17:20:03.066545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:03.250827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518833332892041:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.122326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518832807403743:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:03.154096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518832807403744:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:03.258774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518833332892050:2048] [node 1] ICS09 handshake done sender: [2:7571518833332892040:2118] self: [2:7571518833332892044:0] peer: [1:7571518827568945602:0] socket: 429 qp: -1
2025-11-11T17:20:03.174053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518832807403743:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.269450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518833332892050:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:03.186030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518828512436359:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.274003Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518829037924731:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:03.210528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518832807403746:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:03.290222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:03.225782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518832807403744:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.253888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518828512436358:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.290724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518824742957365:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(65443743ns) {HandleHandshakeStatus:329(65440398ns) {DropIncomingHandshake:441(4872ns) DropOutgoingHandshake:456(14953ns) SwitchToState:178(701ns) ProcessPendingSessionEvents:495(104561ns) {ForwardSessionEventToSession:612(51598ns) ForwardSessionEventToSession:612(3568ns) ForwardSessionEventToSession:612(5394ns) ForwardSessionEventToSession:612(2824ns) ForwardSessionEventToSession:612(10559ns) ForwardSessionEventToSession:612(4780ns) ForwardSessionEventToSession:612(3771ns) ForwardSessionEventToSession:612(7642ns) ForwardSessionEventToSession:612(2680ns) ForwardSessionEventToSession:612(2262ns)}}}
2025-11-11T17:20:03.255773Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518832807403746:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.290913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518829037924739:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.290938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518829037924739:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.269840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518828512436358:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16216393ns) {IncomingHandshake:261(8274234ns) {DropIncomingHandshake:441(963ns) PrepareNewSessionHandshake:476(2757ns) {SwitchToState:178(962ns)} GenerateSessionVirtualId:988(1331ns) IssueIncomingHandshakeReply:193(55450ns) {SwitchToState:178(667ns)}}}
2025-11-11T17:20:03.291446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518824742957365:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:03.293837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518828512436375:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.300924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518829037924725:2104] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.350420Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25234) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:03.300982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:03.361838Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23675) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:03.301025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518829037924725:2104] [node 4] ICP22 created new session: [2:7571518833332892055:2104]
2025-11-11T17:20:03.301087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518833332892055:2104] [node 4] ICS09 handshake done sender: [2:7571518829037924731:2112] self: [2:7571518829037924730:0] peer: [4:7571518832734872010:0] socket: 412 qp: -1
2025-11-11T17:20:03.374123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518828512436358:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25234) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518832807403744:2120] held: no
2025-11-11T17:20:03.301120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518833332892055:2104] [node 4] ICS10 traffic start
2025-11-11T17:20:03.374228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518828512436359:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23675) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518832807403743:2119] held: no
2025-11-11T17:20:03.301175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:03.374486Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:03.306095Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518829037924739:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:03.377529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518832807403749:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:03.338189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518829037924724:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.378053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518828512436375:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518832807403746:2121] held: no
2025-11-11T17:20:03.338271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:03.338314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518829037924724:2102] [node 6] ICP22 created new session: [2:7571518833332892057:2102]
2025-11-11T17:20:03.378196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518832807403750:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:03.338361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518833332892057:2102] [node 6] ICS09 handshake done sender: [2:7571518829037924739:2115] self: [2:7571518829037924738:0] peer: [6:7571518832807403747:0] socket: 434 qp: -1
2025-11-11T17:20:03.378687Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518832807403744:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:03.338389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518833332892057:2102] [node 6] ICS10 traffic start
2025-11-11T17:20:03.338424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:03.378851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518828512436358:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.340385Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518833332892038:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.378928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:03.340407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518833332892038:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.353929Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518833332892038:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:03.378968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518828512436358:2049] [node 2] ICP22 created new session: [6:7571518832807403751:2049]
2025-11-11T17:20:03.393903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518824742957366:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.394024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:03.379009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518832807403751:2049] [node 2] ICS09 handshake done sender: [6:7571518832807403744:2120] self: [6:7571518832807403747:0] peer: [2:7571518829037924738:0] socket: 400 qp: -1
2025-11-11T17:20:03.394084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518824742957366:2049] [node 3] ICP22 created new session: [2:7571518833332892060:2049]
2025-11-11T17:20:03.379037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518832807403751:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:03.379086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:03.394127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518833332892060:2049] [node 3] ICS09 handshake done sender: [2:7571518833332892038:2116] self: [2:7571518833332892037:0] peer: [3:7571518831737653360:0] socket: 411 qp: -1
2025-11-11T17:20:03.379454Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518832807403743:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:03.394169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518833332892060:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:03.379682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518828512436359:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.394233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:03.379734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:03.193896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518834939980910:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:03.214021Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518830645013599:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:03.379766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518828512436359:2050] [node 3] ICP22 created new session: [6:7571518832807403753:2050]
2025-11-11T17:20:03.230511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518834939980912:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9294
2025-11-11T17:20:03.245730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518830645013518:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.379801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518832807403753:2050] [node 3] ICS09 handshake done sender: [6:7571518832807403743:2119] self: [6:7571518832807403745:0] peer: [3:7571518827442686046:0] socket: 428 qp: -1
2025-11-11T17:20:03.274242Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518834939980908:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:03.379827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518832807403753:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:03.286263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:03.379874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:03.305883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518830645013518:2049] [node 2] ICP22 created new session: [5:7571518834939980913:2049]
2025-11-11T17:20:03.325722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518834939980913:2049] [node 2] ICS09 handshake done sender: [5:7571518830645013599:2118] self: [5:7571518834939980909:0] peer: [2:7571518829037924732:0] socket: 415 qp: -1
2025-11-11T17:20:03.341891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518834939980913:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:03.345796Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:03.346666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518830645013517:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.346739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:03.346790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518830645013517:2048] [node 1] ICP22 created new session: [5:7571518834939980915:2048]
2025-11-11T17:20:03.346836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518834939980915:2048] [node 1] ICS09 handshake done sender: [5:7571518834939980908:2121] self: [5:7571518834939980911:0] peer: [1:7571518827568945598:0] socket: 431 qp: -1
2025-11-11T17:20:03.346872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518834939980915:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:03.346930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:03.347233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518830645013517:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:03.353690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:03.326358Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518827442686041:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:03.326593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518827442686033:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.326652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:03.326699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518827442686033:2097] [node 6] ICP22 created new session: [3:7571518831737653356:2097]
2025-11-11T17:20:03.326741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518831737653356:2097] [node 6] ICS09 handshake done sender: [3:7571518827442686047:2119] self: [3:7571518827442686046:0] peer: [6:7571518832807403745:0] socket: 427 qp: -1
2025-11-11T17:20:03.326775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518831737653356:2097] [node 6] ICS10 traffic start
2025-11-11T17:20:03.326811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:03.327644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518827442686036:2095] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.327712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:03.327753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518827442686036:2095] [node 4] ICP22 created new session: [3:7571518831737653358:2095]
2025-11-11T17:20:03.327792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518831737653358:2095] [node 4] ICS09 handshake done sender: [3:7571518827442686041:2116] self: [3:7571518827442686040:0] peer: [4:7571518832734872014:0] socket: 392 qp: -1
2025-11-11T17:20:03.327819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518831737653358:2095] [node 4] ICS10 traffic start
2025-11-11T17:20:03.327843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:03.329991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518831737653353:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-11T17:20:03.339951Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518831737653353:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.340055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518823147718667:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:03.341477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518831737653361:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-11T17:20:03.375594Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518831737653353:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:03.375817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518823147718667:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.375913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:03.375958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518823147718667:2049] [node 2] ICP22 created new session: [3:7571518831737653362:2049]
2025-11-11T17:20:03.375998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518831737653362:2049] [node 2] ICS09 handshake done sender: [3:7571518831737653353:2120] self: [3:7571518831737653360:0] peer: [2:7571518833332892037:0] socket: 426 qp: -1
2025-11-11T17:20:03.376033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518831737653362:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:03.376102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:03.407821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518830645013518:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(171840970ns) {HandleHandshakeStatus:329(171838387ns) {DropIncomingHandshake:441(2801ns) DropOutgoingHandshake:456(21471ns) SwitchToState:178(416ns) ProcessPendingSessionEvents:495(15908367ns) {ForwardSessionEventToSession:612(66869ns) ForwardSessionEventToSession:612(7583ns) ForwardSessionEventToSession:612(2477ns) ForwardSessionEventToSession:612(4762ns) ForwardSessionEventToSession:612(7843937ns) ForwardSessionEventToSession:612(12149ns) ForwardSessionEventToSession:612(12967ns) ForwardSessionEventToSession:612(2734ns) ForwardSessionEventToSession:612(2779ns) ForwardSessionEventToSession:612(7932571ns)}}}
2025-11-11T17:20:03.248120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518828439904623:2048] [node 1] ICP22 created new session: [4:7571518832734872012:2048]
2025-11-11T17:20:03.248167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518832734872012:2048] [node 1] ICS09 handshake done sender: [4:7571518828439904690:2111] self: [4:7571518828439904706:0] peer: [1:7571518827568945593:0] socket: 399 qp: -1
2025-11-11T17:20:03.248203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518832734872012:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:03.248250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:03.250357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518828439904625:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:03.281701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518832734872011:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:03.306203Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518828439904701:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:03.306433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518828439904624:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.306508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:03.146079Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518831863912912:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.306561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518828439904624:2049] [node 2] ICP22 created new session: [4:7571518832734872015:2049]
2025-11-11T17:20:03.161939Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945604:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.306600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518832734872015:2049] [node 2] ICS09 handshake done sender: [4:7571518828439904701:2116] self: [4:7571518832734872010:0] peer: [2:7571518829037924730:0] socket: 416 qp: -1
2025-11-11T17:20:03.161979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518827568945604:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.306632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518832734872015:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:03.169846Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518827568945604:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:03.306668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:03.205885Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518827568945599:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.339224Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23675) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:03.205911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518827568945599:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.341795Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518828439904704:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:03.206373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518823273978226:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.341826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518828439904704:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:03.206430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:03.349928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518828439904625:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23675) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518832734872006:2120] held: no
2025-11-11T17:20:03.206480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518823273978226:2048] [node 2] ICP22 created new session: [1:7571518831863912917:2048]
2025-11-11T17:20:03.357957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518832734872017:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20283
2025-11-11T17:20:03.206524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518831863912917:2048] [node 2] ICS09 handshake done sender: [1:7571518827568945604:2116] self: [1:7571518827568945602:0] peer: [2:7571518833332892044:0] socket: 425 qp: -1
2025-11-11T17:20:03.359228Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518828439904704:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:03.206558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518831863912917:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:03.359462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518828439904676:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.206593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:03.359544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:03.233946Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518827568945599:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:03.359603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518828439904676:2067] [node 6] ICP22 created new session: [4:7571518832734872019:2067]
2025-11-11T17:20:03.297700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518827568945586:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:03.297797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:03.359654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518832734872019:2067] [node 6] ICS09 handshake done sender: [4:7571518828439904704:2117] self: [4:7571518828439904702:0] peer: [6:7571518832807403748:0] socket: 438 qp: -1
2025-11-11T17:20:03.297851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518827568945586:2107] [node 5] ICP22 created new session: [1:7571518831863912920:2107]
2025-11-11T17:20:03.359703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518832734872019:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:03.297900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518831863912920:2107] [node 5] ICS09 handshake done sender: [1:7571518827568945599:2114] self: [1:7571518827568945598:0] peer: [5:7571518834939980911:0] socket: 388 qp: -1
2025-11-11T17:20:03.359735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:03.297936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518831863912920:2107] [node 5] ICS10 traffic start
2025-11-11T17:20:03.360747Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518832734872006:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:03.297967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:03.360958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518828439904625:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.361017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:03.361055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518828439904625:2050] [node 3] ICP22 created new session: [4:7571518832734872021:2050]
2025-11-11T17:20:03.361095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518832734872021:2050] [node 3] ICS09 handshake done sender: [4:7571518832734872006:2120] self: [4:7571518832734872014:0] peer: [3:7571518827442686040:0] socket: 433 qp: -1
2025-11-11T17:20:03.361127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518832734872021:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:03.361159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:03.409871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518832807403755:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41047
2025-11-11T17:20:03.414136Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518832807403746:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:03.414435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518828512436375:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:03.414517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:03.414565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518828512436375:2063] [node 4] ICP22 created new session: [6:7571518832807403756:2063]
2025-11-11T17:20:03.414608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518832807403756:2063] [node 4] ICS09 handshake done sender: [6:7571518832807403746:2121] self: [6:7571518832807403748:0] peer: [4:7571518828439904702:0] socket: 439 qp: -1
2025-11-11T17:20:03.414646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518832807403756:2063] [node 4] ICS10 traffic start
2025-11-11T17:20:03.414681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:03.517851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:03.518106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518833332892050:2048] [node 1] ICS01 socket: 429 reason# EndOfStream
2025-11-11T17:20:03.518148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518824742957365:2048] [node 1] ICP30 unregister session Session# [2:7571518833332892050:2048] VirtualId# [2:7571518833332892044:0]
2025-11-11T17:20:03.518185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518833332892050:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.518288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:03.518329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:03.529938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:03.530184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518831737653348:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-11T17:20:03.530218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518823147718666:2048] [node 1] ICP30 unregister session Session# [3:7571518831737653348:2048] VirtualId# [3:7571518827442686042:0]
2025-11-11T17:20:03.530247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518831737653348:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.530335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:03.530373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:03.541807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:03.542064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518834939980915:2048] [node 1] ICS01 socket: 431 reason# EndOfStream
2025-11-11T17:20:03.542101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518830645013517:2048] [node 1] ICP30 unregister session Session# [5:7571518834939980915:2048] VirtualId# [5:7571518834939980911:0]
2025-11-11T17:20:03.542134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518834939980915:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.542234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:03.542282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:03.542361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:03.542549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518832734872012:2048] [node 1] ICS01 socket: 399 reason# EndOfStream
2025-11-11T17:20:03.542582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518828439904623:2048] [node 1] ICP30 unregister session Session# [4:7571518832734872012:2048] VirtualId# [4:7571518828439904706:0]
2025-11-11T17:20:03.542601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518832734872012:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.542641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:03.542673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:03.543508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:03.543761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518828512436440:2048] [node 1] ICS01 socket: 396 reason# EndOfStream
2025-11-11T17:20:03.543818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518828512436357:2048] [node 1] ICP30 unregister session Session# [6:7571518828512436440:2048] VirtualId# [6:7571518828512436436:0]
2025-11-11T17:20:03.543847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518828512436440:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.543933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:03.543976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:03.639910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:03.640159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518831737653362:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-11-11T17:20:03.640197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518823147718667:2049] [node 2] ICP30 unregister session Session# [3:7571518831737653362:2049] VirtualId# [3:7571518831737653360:0]
2025-11-11T17:20:03.640231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518831737653362:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.640318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:03.640356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:03.649802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:03.650072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518832734872015:2049] [node 2] ICS01 socket: 416 reason# EndOfStream
2025-11-11T17:20:03.650115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518828439904624:2049] [node 2] ICP30 unregister session Session# [4:7571518832734872015:2049] VirtualId# [4:7571518832734872010:0]
2025-11-11T17:20:03.650145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518832734872015:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.650226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:03.649860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:03.650265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:03.650118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518832807403751:2049] [node 2] ICS01 socket: 400 reason# EndOfStream
2025-11-11T17:20:03.650160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518828512436358:2049] [node 2] ICP30 unregister session Session# [6:7571518832807403751:2049] VirtualId# [6:7571518832807403747:0]
2025-11-11T17:20:03.650194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518832807403751:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.650304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:03.650357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:03.652636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:03.652870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518834939980913:2049] [node 2] ICS07 socket disconnect 415 reason# EndOfStream
2025-11-11T17:20:03.652903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518834939980913:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.652986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:03.653030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518834939980913:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:03.657828Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25234) Couldn't connect to any resolved address
2025-11-11T17:20:03.658107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518830645013518:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25234) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:03.658161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518834939980913:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:03.658194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518830645013518:2049] [node 2] ICP30 unregister session Session# [5:7571518834939980913:2049] VirtualId# [5:7571518834939980909:0]
2025-11-11T17:20:03.658236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:03.658265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518830645013518:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25234) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:03.658279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25234) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:03.669728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518830645013518:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:03.777814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:03.778047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518834939980900:2050] [node 3] ICS01 socket: 381 reason# EndOfStream
2025-11-11T17:20:03.778089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518830645013519:2050] [node 3] ICP30 unregister session Session# [5:7571518834939980900:2050] VirtualId# [5:7571518830645013593:0]
2025-11-11T17:20:03.778120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518834939980900:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.778200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:03.778239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:03.781709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:03.781934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518832734872021:2050] [node 3] ICS01 socket: 433 reason# EndOfStream
2025-11-11T17:20:03.781966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518828439904625:2050] [node 3] ICP30 unregister session Session# [4:7571518832734872021:2050] VirtualId# [4:7571518832734872014:0]
2025-11-11T17:20:03.782002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518832734872021:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.782092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:03.782136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:03.781713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:03.781938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518832807403753:2050] [node 3] ICS01 socket: 428 reason# EndOfStream
2025-11-11T17:20:03.781971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518828512436359:2050] [node 3] ICP30 unregister session Session# [6:7571518832807403753:2050] VirtualId# [6:7571518832807403745:0]
2025-11-11T17:20:03.782003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518832807403753:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.782092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:03.782138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:03.837797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:03.838017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518834939980905:2064] [node 4] ICS01 socket: 408 reason# EndOfStream
2025-11-11T17:20:03.838061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518830645013535:2064] [node 4] ICP30 unregister session Session# [5:7571518834939980905:2064] VirtualId# [5:7571518830645013600:0]
2025-11-11T17:20:03.838095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518834939980905:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.838186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:03.838227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:03.841709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:03.842386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518832807403756:2063] [node 4] ICS01 socket: 439 reason# EndOfStream
2025-11-11T17:20:03.842425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518828512436375:2063] [node 4] ICP30 unregister session Session# [6:7571518832807403756:2063] VirtualId# [6:7571518832807403748:0]
2025-11-11T17:20:03.842457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518832807403756:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.842567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:03.842612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:03.969789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:03.970066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518832807403740:2066] [node 5] ICS01 socket: 397 reason# EndOfStream
2025-11-11T17:20:03.970102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518828512436406:2066] [node 5] ICP30 unregister session Session# [6:7571518832807403740:2066] VirtualId# [6:7571518828512436435:0]
2025-11-11T17:20:03.970134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518832807403740:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:03.970261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:03.970306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:04.249904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518836727885696:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:04.249964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518836727885697:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:04.301907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518838563063294:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:04.301955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518838563063295:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:04.337937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518838129832719:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:04.337984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518838129832720:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:04.413911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518837026349028:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:04.413961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518837026349029:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:04.413988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518837026349030:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:04.465933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518839699060590:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:04.465994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518839699060592:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:04.466060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518839699060591:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:04.510027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518838929271592:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:04.510073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518838929271593:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:04.510099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518838929271594:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:04.890457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518838929271608:2062] [node 5] ICP01 ready to work
2025-11-11T17:20:04.854360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518836727885751:2101] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:04.859378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518838129832744:2071] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:04.861607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518836727885757:2105] [node 6] ICP01 ready to work
2025-11-11T17:20:05.033332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518841321316338:2060] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:04.968176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518839699060590:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(78131437ns) {Configure:104(78122231ns) {ProcessConfigured:135(81844ns) {StartInitialHandshake:157(78599ns) {DropHandshakes:469(5062ns) {DropIncomingHandshake:441(579ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(953ns) PrepareNewSessionHandshake:476(2541ns) {SwitchToState:178(1209ns)}}}}}
2025-11-11T17:20:04.946234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518838929271612:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:04.970584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518836727885696:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(114173744ns) {Configure:104(114165181ns) {ProcessConfigured:135(20626829ns) {StartInitialHandshake:157(20622998ns) {DropHandshakes:469(3816ns) {DropIncomingHandshake:441(799ns) DropOutgoingHandshake:456(626ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:476(3988ns) {SwitchToState:178(1750ns)}}}}}
2025-11-11T17:20:05.037229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518841321316344:2063] [node 6] ICP01 ready to work
2025-11-11T17:20:04.968688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518839699060613:2067] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:04.965803Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518838929271612:2064] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.046208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518841022853066:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.000625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518839699060617:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.065650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518843224238912:2067] [node 4] ICP01 ready to work
2025-11-11T17:20:05.147407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316378:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.125714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518842858030654:2090] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.046434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518841022853067:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.147593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316393:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.000792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518839699060618:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.209617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518842424800079:2072] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.135589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518838563063295:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(160506565ns) {Configure:104(160495310ns) {ProcessConfigured:135(151802ns) {StartInitialHandshake:157(148466ns) {DropHandshakes:469(3321ns) {DropIncomingHandshake:441(814ns) DropOutgoingHandshake:456(647ns)} GenerateSessionVirtualId:988(1020ns) PrepareNewSessionHandshake:476(2464ns) {SwitchToState:178(1005ns)}}}}}
2025-11-11T17:20:05.266123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518838929271594:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(412441507ns) {Configure:104(412435641ns) {ProcessConfigured:135(108652ns) {StartInitialHandshake:157(106023ns) {DropHandshakes:469(3671ns) {DropIncomingHandshake:441(985ns) DropOutgoingHandshake:456(728ns)} GenerateSessionVirtualId:988(904ns) PrepareNewSessionHandshake:476(2828ns) {SwitchToState:178(865ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.116522Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518836727885753:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.005866Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518839699060617:2072] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.198022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518842858030658:2089] [node 6] ICP01 ready to work
2025-11-11T17:20:05.250086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518842424800082:2073] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.270213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.116569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518836727885753:2102] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.186132Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518841321316393:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.005947Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518839699060618:2073] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.149826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518841022853066:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.153977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518836727885753:2102] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:05.198160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518841022853069:2103] [node 4] ICP01 ready to work
2025-11-11T17:20:05.202554Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518841022853067:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.202598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518836727885751:2101] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.202657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:05.202709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518836727885751:2101] [node 5] ICP22 created new session: [1:7571518841022853071:2101]
2025-11-11T17:20:05.202754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518841022853071:2101] [node 5] ICS09 handshake done sender: [1:7571518836727885753:2102] self: [1:7571518836727885752:0] peer: [5:7571518843994027919:0] socket: 385 qp: -1
2025-11-11T17:20:05.202790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518841022853071:2101] [node 5] ICS10 traffic start
2025-11-11T17:20:05.202826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:05.234879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518841022853073:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-11-11T17:20:05.274471Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518836727885767:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.274498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518836727885767:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.277971Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518836727885767:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:05.278410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518836727885696:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.278489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:05.278535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518836727885696:2048] [node 2] ICP22 created new session: [1:7571518841022853075:2048]
2025-11-11T17:20:05.278589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518841022853075:2048] [node 2] ICS09 handshake done sender: [1:7571518836727885767:2112] self: [1:7571518836727885765:0] peer: [2:7571518842858030675:0] socket: 389 qp: -1
2025-11-11T17:20:05.278623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518841022853075:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:05.278659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:05.303342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518836727885697:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(393438018ns) {Configure:104(393431011ns) {ProcessConfigured:135(80106ns) {StartInitialHandshake:157(78161ns) {DropHandshakes:469(2831ns) {DropIncomingHandshake:441(610ns) DropOutgoingHandshake:456(618ns)} GenerateSessionVirtualId:988(1509ns) PrepareNewSessionHandshake:476(1864ns) {SwitchToState:178(625ns)}}}}}
2025-11-11T17:20:05.304521Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518841022853073:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.306302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518836727885696:2048] [node 2] ICP18 handshake (actor [1:7571518841022853073:2115]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.211504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518842858030654:2090] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(75451040ns) {Configure:104(75444323ns) {ProcessConfigured:135(13173236ns) {StartInitialHandshake:157(13169925ns) {DropHandshakes:469(3466ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(757ns)} GenerateSessionVirtualId:988(901ns) PrepareNewSessionHandshake:476(4229ns) {SwitchToState:178(2135ns)}}}}}
2025-11-11T17:20:05.255962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518838129832744:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(322175108ns) {Configure:104(322160887ns) {ProcessConfigured:135(64677836ns) {StartInitialHandshake:157(64673022ns) {DropHandshakes:469(2519ns) {DropIncomingHandshake:441(504ns) DropOutgoingHandshake:456(459ns)} GenerateSessionVirtualId:988(953ns) PrepareNewSessionHandshake:476(3678ns) {SwitchToState:178(1320ns)}}}}}
2025-11-11T17:20:05.007384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518839699060590:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.186210Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518841321316378:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.254825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518841321316344:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(217324562ns) {Configure:104(217320216ns) {ProcessConfigured:135(81581ns) {StartInitialHandshake:157(78924ns) {DropHandshakes:469(3261ns) {DropIncomingHandshake:441(819ns) DropOutgoingHandshake:456(537ns)} GenerateSessionVirtualId:988(678ns) PrepareNewSessionHandshake:476(1920ns) {SwitchToState:178(726ns)}}}}}
2025-11-11T17:20:05.255728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316401:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.315717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.319324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518841321316338:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(269205407ns) {Configure:104(269197923ns) {ProcessConfigured:135(88178ns) {StartInitialHandshake:157(86003ns) {DropHandshakes:469(3117ns) {DropIncomingHandshake:441(661ns) DropOutgoingHandshake:456(445ns)} GenerateSessionVirtualId:988(1167ns) PrepareNewSessionHandshake:476(1561ns) {SwitchToState:178(500ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.339022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518841321316401:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.345888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518837026349029:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.054793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518839699060592:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(85747879ns) {Configure:104(85743171ns) {ProcessConfigured:135(52235ns) {StartInitialHandshake:157(48795ns) {DropHandshakes:469(2121ns) {DropIncomingHandshake:441(525ns) DropOutgoingHandshake:456(392ns)} GenerateSessionVirtualId:988(856ns) PrepareNewSessionHandshake:476(1237ns) {SwitchToState:178(441ns)}}}}}
2025-11-11T17:20:05.214131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518842858030657:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43357
2025-11-11T17:20:05.312039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518838129832720:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(78198364ns) {Configure:104(78188901ns) {ProcessConfigured:135(104632ns) {StartInitialHandshake:157(76586ns) {DropHandshakes:469(3219ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(659ns)} GenerateSessionVirtualId:988(766ns) PrepareNewSessionHandshake:476(2264ns) {SwitchToState:178(758ns)}}}}}
2025-11-11T17:20:05.168380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518843994027960:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.168861Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518839699060618:2073] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:05.169031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518839699060590:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.169105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:05.169159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518839699060590:2048] [node 1] ICP22 created new session: [5:7571518843994027961:2048]
2025-11-11T17:20:05.169201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518843994027961:2048] [node 1] ICS09 handshake done sender: [5:7571518839699060618:2073] self: [5:7571518843994027919:0] peer: [1:7571518836727885752:0] socket: 390 qp: -1
2025-11-11T17:20:05.169237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518843994027961:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:05.169279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:05.225883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518843994027965:2064] [node 6] ICP01 ready to work
2025-11-11T17:20:05.313107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518843994027966:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.338660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518843994027971:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.361530Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518843994027966:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.339125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238966:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.379297Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518843994027968:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.379332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518843994027968:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.380590Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518843994027968:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:05.214383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518842858030662:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43357
2025-11-11T17:20:05.381399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518843994027965:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.398164Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518841321316402:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.399951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518839699060591:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(474467115ns) {Configure:104(474456812ns) {ProcessConfigured:135(33971621ns) {StartInitialHandshake:157(29957541ns) {DropHandshakes:469(5117ns) {DropIncomingHandshake:441(458ns) DropOutgoingHandshake:456(884ns)} GenerateSessionVirtualId:988(2339ns) PrepareNewSessionHandshake:476(2561ns) {SwitchToState:178(688ns)}}}}}
2025-11-11T17:20:05.400177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518839699060591:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.400865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316407:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.312741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518842424800088:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3135
2025-11-11T17:20:05.330705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518838129832719:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(380587757ns) {Configure:104(380583125ns) {ProcessConfigured:135(78398ns) {StartInitialHandshake:157(76119ns) {DropHandshakes:469(3056ns) {DropIncomingHandshake:441(586ns) DropOutgoingHandshake:456(645ns)} GenerateSessionVirtualId:988(1294ns) PrepareNewSessionHandshake:476(5342ns) {SwitchToState:178(886ns)}}}}}
2025-11-11T17:20:05.338071Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800088:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.365774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518842424800093:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3135
2025-11-11T17:20:05.406178Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800093:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.214494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518842858030663:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43357
2025-11-11T17:20:05.341833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518843224238965:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.422837Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518841022853078:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.422870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518841022853078:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.402053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518843994027971:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.413261Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518841321316398:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.221893Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030663:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.356620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238967:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.421842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518839699060592:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.413331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518841321316398:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.421996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518838129832719:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.444957Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800080:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.444984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518842424800080:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.478036Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518842424800080:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.462220Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518841022853078:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.222015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030662:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.378846Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518843224238966:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.422562Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518841321316401:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:05.438864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518839699060592:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(24741295ns) {IncomingHandshake:261(24739198ns) {DropIncomingHandshake:441(488ns) PrepareNewSessionHandshake:476(1021ns) {SwitchToState:178(502ns)} GenerateSessionVirtualId:988(325ns) IssueIncomingHandshakeReply:193(16895758ns) {SwitchToState:178(5231ns)}}}
2025-11-11T17:20:05.504372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518842424800099:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3135
2025-11-11T17:20:05.378998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518838929271608:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.486159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518836727885759:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.229780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518842858030660:2088] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.465729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:05.424010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518837026349029:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.380389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238970:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.486204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518836727885759:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.262021Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030657:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.520027Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800099:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.424108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:05.465800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518843994027965:2064] [node 6] ICP22 created new session: [5:7571518843994027978:2064]
2025-11-11T17:20:05.274070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518838563063294:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.503131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518836727885697:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.520128Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800084:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.381154Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518843224238966:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:05.424150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518837026349029:2049] [node 2] ICP22 created new session: [4:7571518841321316408:2049]
2025-11-11T17:20:05.467395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518843994027979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.276291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518842858030676:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43357
2025-11-11T17:20:05.358082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030669:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.358114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518842858030669:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.358674Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518842858030657:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:05.361822Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:05.366356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518838563063294:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.366429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:05.366481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518838563063294:2048] [node 1] ICP22 created new session: [2:7571518842858030678:2048]
2025-11-11T17:20:05.366531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518842858030678:2048] [node 1] ICS09 handshake done sender: [2:7571518842858030657:2112] self: [2:7571518842858030675:0] peer: [1:7571518836727885765:0] socket: 402 qp: -1
2025-11-11T17:20:05.366567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518842858030678:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:05.366621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:05.366962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518838563063294:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:05.370819Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518842858030669:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:05.418288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518842858030680:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43357
2025-11-11T17:20:05.421922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518842858030660:2088] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.433943Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030661:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.433987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518842858030661:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.459887Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518842858030661:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:05.461210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518842858030654:2090] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.461252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:05.461296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518842858030654:2090] [node 5] ICP22 created new session: [2:7571518842858030684:2090]
2025-11-11T17:20:05.461340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518842858030684:2090] [node 5] ICS09 handshake done sender: [2:7571518842858030661:2113] self: [2:7571518842858030659:0] peer: [5:7571518843994027975:0] socket: 406 qp: -1
2025-11-11T17:20:05.461374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518842858030684:2090] [node 5] ICS10 traffic start
2025-11-11T17:20:05.461433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:05.482054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030680:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.497708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:05.505269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518842858030660:2088] [node 4] ICP22 created new session: [2:7571518842858030686:2088]
2025-11-11T17:20:05.525726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518842858030686:2088] [node 4] ICS09 handshake done sender: [2:7571518842858030669:2117] self: [2:7571518842858030668:0] peer: [4:7571518841321316406:0] socket: 408 qp: -1
2025-11-11T17:20:05.538218Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030656:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.538255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518842858030656:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.541764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518842858030686:2088] [node 4] ICS10 traffic start
2025-11-11T17:20:05.550718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518842858030654:2090] [node 5] ICP35 event processing took too much time PendingConnection:278(25221872ns) {HandleHandshakeStatus:329(25218412ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(763ns) SwitchToState:178(911ns) ProcessPendingSessionEvents:495(24867494ns) {ForwardSessionEventToSession:612(67173ns) ForwardSessionEventToSession:612(3446ns) ForwardSessionEventToSession:612(10286406ns) ForwardSessionEventToSession:612(19107ns) ForwardSessionEventToSession:612(6074924ns) ForwardSessionEventToSession:612(4282ns) ForwardSessionEventToSession:612(2961ns) ForwardSessionEventToSession:612(8377804ns) ForwardSessionEventToSession:612(4511ns) ForwardSessionEventToSession:612(2939ns)}}}
2025-11-11T17:20:05.551118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518842858030654:2090] [node 5] ICP18 handshake (actor [2:7571518842858030680:2120]) is too old
2025-11-11T17:20:05.561953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518842858030688:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43357
2025-11-11T17:20:05.570221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:05.578942Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518842858030656:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:05.617670Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518842858030673:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.617712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518842858030673:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.621864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518838563063295:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.621945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:05.621997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518838563063295:2049] [node 3] ICP22 created new session: [2:7571518842858030690:2049]
2025-11-11T17:20:05.622045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518842858030690:2049] [node 3] ICS09 handshake done sender: [2:7571518842858030656:2111] self: [2:7571518842858030655:0] peer: [3:7571518842424800104:0] socket: 430 qp: -1
2025-11-11T17:20:05.622081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518842858030690:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:05.622115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:05.623019Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:05.520148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518842424800084:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.503221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:05.503270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518836727885697:2049] [node 3] ICP22 created new session: [1:7571518841022853081:2049]
2025-11-11T17:20:05.503313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518841022853081:2049] [node 3] ICS09 handshake done sender: [1:7571518841022853078:2116] self: [1:7571518841022853077:0] peer: [3:7571518842424800097:0] socket: 391 qp: -1
2025-11-11T17:20:05.503352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518841022853081:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:05.503391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:05.515243Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518836727885759:2107] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:05.522958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518841022853085:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-11-11T17:20:05.528302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518836727885757:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.528388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:05.528439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518836727885757:2105] [node 6] ICP22 created new session: [1:7571518841022853086:2105]
2025-11-11T17:20:05.528489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518841022853086:2105] [node 6] ICS09 handshake done sender: [1:7571518836727885759:2107] self: [1:7571518836727885758:0] peer: [6:7571518843224238977:0] socket: 386 qp: -1
2025-11-11T17:20:05.528525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518841022853086:2105] [node 6] ICS10 traffic start
2025-11-11T17:20:05.528565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:05.545999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518841022853069:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(311588530ns) {Configure:104(311582991ns) {ProcessConfigured:135(31969670ns) {StartInitialHandshake:157(31966532ns) {DropHandshakes:469(2729ns) {DropIncomingHandshake:441(794ns) DropOutgoingHandshake:456(513ns)} GenerateSessionVirtualId:988(654ns) PrepareNewSessionHandshake:476(4210ns) {SwitchToState:178(1823ns)}}}}}
2025-11-11T17:20:05.601938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518841022853085:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.627246Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518841022853088:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.627268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518841022853088:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.629354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518836727885757:2105] [node 6] ICP18 handshake (actor [1:7571518841022853085:2117]) is too old
2025-11-11T17:20:05.633768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518841022853088:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.467504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518843994027980:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.481893Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518843994027966:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:05.489656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518843994027978:2064] [node 6] ICS09 handshake done sender: [5:7571518843994027968:2117] self: [5:7571518843994027967:0] peer: [6:7571518843224238969:0] socket: 418 qp: -1
2025-11-11T17:20:05.513990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518843994027978:2064] [node 6] ICS10 traffic start
2025-11-11T17:20:05.514069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:05.514586Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518843994027965:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(132948310ns) {HandleHandshakeStatus:329(132945087ns) {DropIncomingHandshake:441(8339550ns) DropOutgoingHandshake:456(1189ns) SwitchToState:178(4511ns) ProcessPendingSessionEvents:495(113650ns) {ForwardSessionEventToSession:612(59317ns) ForwardSessionEventToSession:612(3721ns) ForwardSessionEventToSession:612(5894ns) ForwardSessionEventToSession:612(3298ns) ForwardSessionEventToSession:612(2850ns) ForwardSessionEventToSession:612(8755ns) ForwardSessionEventToSession:612(3879ns) ForwardSessionEventToSession:612(2736ns) ForwardSessionEventToSession:612(5151ns) ForwardSessionEventToSession:612(3098ns)}}}
2025-11-11T17:20:05.514712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518839699060591:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.514779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:05.514828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518839699060591:2049] [node 2] ICP22 created new session: [5:7571518843994027983:2049]
2025-11-11T17:20:05.514867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518843994027983:2049] [node 2] ICS09 handshake done sender: [5:7571518843994027966:2116] self: [5:7571518843994027975:0] peer: [2:7571518842858030659:0] socket: 411 qp: -1
2025-11-11T17:20:05.514913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518843994027983:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:05.514943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:05.525658Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518843994027971:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:05.590055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518839699060591:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(75045249ns) {HandleHandshakeStatus:329(75040047ns) {DropIncomingHandshake:441(646ns) DropOutgoingHandshake:456(4910ns) SwitchToState:178(366ns) ProcessPendingSessionEvents:495(21918336ns) {ForwardSessionEventToSession:612(21864505ns) ForwardSessionEventToSession:612(7130ns) ForwardSessionEventToSession:612(8544ns) ForwardSessionEventToSession:612(2570ns) ForwardSessionEventToSession:612(3027ns) ForwardSessionEventToSession:612(4389ns) ForwardSessionEventToSession:612(3178ns) ForwardSessionEventToSession:612(4922ns) ForwardSessionEventToSession:612(5196ns) ForwardSessionEventToSession:612(2819ns)}}}
2025-11-11T17:20:05.590126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518839699060592:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.590219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:05.590265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518839699060592:2050] [node 3] ICP22 created new session: [5:7571518843994027985:2050]
2025-11-11T17:20:05.590309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518843994027985:2050] [node 3] ICS09 handshake done sender: [5:7571518843994027971:2118] self: [5:7571518843994027977:0] peer: [3:7571518842424800078:0] socket: 423 qp: -1
2025-11-11T17:20:05.590346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518843994027985:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:05.590403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:05.591550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518843994027982:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.599496Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:05.601719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518839699060592:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:05.628759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518838563063295:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:05.653704Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518843994027982:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.381507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518838929271608:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.520759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518838129832744:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.424208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518841321316408:2049] [node 2] ICS09 handshake done sender: [4:7571518841321316401:2115] self: [4:7571518841321316406:0] peer: [2:7571518842858030668:0] socket: 410 qp: -1
2025-11-11T17:20:05.685712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518839699060613:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.520857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:05.630265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518842858030660:2088] [node 4] ICP35 event processing took too much time PendingConnection:278(208087334ns) {HandleHandshakeStatus:329(208084945ns) {DropIncomingHandshake:441(29796ns) DropOutgoingHandshake:456(656ns) SwitchToState:178(651ns) ProcessPendingSessionEvents:495(4305431ns) {ForwardSessionEventToSession:612(4254230ns) ForwardSessionEventToSession:612(4666ns) ForwardSessionEventToSession:612(11748ns) ForwardSessionEventToSession:612(2968ns) ForwardSessionEventToSession:612(2065ns) ForwardSessionEventToSession:612(5010ns) ForwardSessionEventToSession:612(3250ns) ForwardSessionEventToSession:612(5302ns) ForwardSessionEventToSession:612(2639ns) ForwardSessionEventToSession:612(2562ns)}}}
2025-11-11T17:20:05.652673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518841022853069:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.381570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:05.424246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518841321316408:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:05.424304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:05.713740Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:05.645988Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518842858030673:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:05.646344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518842858030658:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.646447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:05.646491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518842858030658:2089] [node 6] ICP22 created new session: [2:7571518842858030693:2089]
2025-11-11T17:20:05.646530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518842858030693:2089] [node 6] ICS09 handshake done sender: [2:7571518842858030673:2118] self: [2:7571518842858030672:0] peer: [6:7571518843224238984:0] socket: 384 qp: -1
2025-11-11T17:20:05.381622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518838929271608:2062] [node 5] ICP22 created new session: [6:7571518843224238971:2062]
2025-11-11T17:20:05.723297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518843994027988:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39691
2025-11-11T17:20:05.520923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518838129832744:2071] [node 5] ICP22 created new session: [3:7571518842424800100:2071]
2025-11-11T17:20:05.429041Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518841321316398:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:05.445976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518841321316344:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.446079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:05.446148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518841321316344:2063] [node 6] ICP22 created new session: [4:7571518841321316411:2063]
2025-11-11T17:20:05.446199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518841321316411:2063] [node 6] ICS09 handshake done sender: [4:7571518841321316398:2114] self: [4:7571518841321316397:0] peer: [6:7571518843224238968:0] socket: 412 qp: -1
2025-11-11T17:20:05.446237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518841321316411:2063] [node 6] ICS10 traffic start
2025-11-11T17:20:05.446292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:05.510459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518837026349030:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(381604210ns) {Configure:104(381592872ns) {ProcessConfigured:135(15572350ns) {StartInitialHandshake:157(15568921ns) {DropHandshakes:469(3233ns) {DropIncomingHandshake:441(709ns) DropOutgoingHandshake:456(485ns)} GenerateSessionVirtualId:988(1039ns) PrepareNewSessionHandshake:476(3849ns) {SwitchToState:178(1550ns)}}}}}
2025-11-11T17:20:05.510672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518837026349030:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.562407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316418:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.581636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316419:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.592293Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518841321316402:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:05.613741Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518841321316419:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.617687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518837026349030:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.625878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518837026349028:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.635661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518841321316421:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9715
2025-11-11T17:20:05.636148Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518841321316419:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:05.636871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518837026349028:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.636972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:05.637030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518837026349028:2048] [node 1] ICP22 created new session: [4:7571518841321316422:2048]
2025-11-11T17:20:05.637075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518841321316422:2048] [node 1] ICS09 handshake done sender: [4:7571518841321316419:2121] self: [4:7571518841321316420:0] peer: [1:7571518841022853084:0] socket: 452 qp: -1
2025-11-11T17:20:05.637111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518841321316422:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:05.637165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:05.645683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:05.645740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518837026349030:2050] [node 3] ICP22 created new session: [4:7571518841321316424:2050]
2025-11-11T17:20:05.677803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518841321316424:2050] [node 3] ICS09 handshake done sender: [4:7571518841321316402:2116] self: [4:7571518841321316416:0] peer: [3:7571518842424800083:0] socket: 422 qp: -1
2025-11-11T17:20:05.677854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518841321316424:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:05.677898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:05.678349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518837026349030:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(67920338ns) {HandleHandshakeStatus:329(60464094ns) {DropIncomingHandshake:441(1047ns) DropOutgoingHandshake:456(7711ns) SwitchToState:178(614ns) ProcessPendingSessionEvents:495(88800ns) {ForwardSessionEventToSession:612(36436ns) ForwardSessionEventToSession:612(5929ns) ForwardSessionEventToSession:612(2984ns) ForwardSessionEventToSession:612(4884ns) ForwardSessionEventToSession:612(7064ns) ForwardSessionEventToSession:612(3511ns) ForwardSessionEventToSession:612(5924ns) ForwardSessionEventToSession:612(2894ns) ForwardSessionEventToSession:612(2983ns) ForwardSessionEventToSession:612(8910ns)}}}
2025-11-11T17:20:05.693713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518841321316404:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.693744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518841321316404:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.694820Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518841321316404:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:05.695057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518841321316338:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.695125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:05.695169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518841321316338:2060] [node 5] ICP22 created new session: [4:7571518841321316427:2060]
2025-11-11T17:20:05.695216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518841321316427:2060] [node 5] ICS09 handshake done sender: [4:7571518841321316404:2117] self: [4:7571518841321316403:0] peer: [5:7571518843994027987:0] socket: 441 qp: -1
2025-11-11T17:20:05.695244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518841321316427:2060] [node 5] ICS10 traffic start
2025-11-11T17:20:05.695293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:05.717757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:05.737828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518841022853069:2103] [node 4] ICP22 created new session: [1:7571518841022853092:2103]
2025-11-11T17:20:05.646561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518842858030693:2089] [node 6] ICS10 traffic start
2025-11-11T17:20:05.646629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:05.520962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518842424800100:2071] [node 5] ICS09 handshake done sender: [3:7571518842424800080:2110] self: [3:7571518842424800078:0] peer: [5:7571518843994027977:0] socket: 420 qp: -1
2025-11-11T17:20:05.520998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518842424800100:2071] [node 5] ICS10 traffic start
2025-11-11T17:20:05.521030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:05.522096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518838129832720:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.381661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518843224238971:2062] [node 5] ICS09 handshake done sender: [6:7571518843224238966:2116] self: [6:7571518843224238969:0] peer: [5:7571518843994027967:0] socket: 419 qp: -1
2025-11-11T17:20:05.749846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518839699060613:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518843994027982:2122] held: no
2025-11-11T17:20:05.522555Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518842424800084:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:05.558405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518842424800105:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3135
2025-11-11T17:20:05.565984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518842424800106:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3135
2025-11-11T17:20:05.566156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518842424800107:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3135
2025-11-11T17:20:05.566211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518842424800108:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3135
2025-11-11T17:20:05.567912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518842424800079:2072] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.594381Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518842424800093:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:05.594895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518838129832719:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.594987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:05.595051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518838129832719:2048] [node 1] ICP22 created new session: [3:7571518842424800109:2048]
2025-11-11T17:20:05.595114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518842424800109:2048] [node 1] ICS09 handshake done sender: [3:7571518842424800093:2115] self: [3:7571518842424800097:0] peer: [1:7571518841022853077:0] socket: 417 qp: -1
2025-11-11T17:20:05.595150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518842424800109:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:05.595181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:05.595792Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518842424800099:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:05.596318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518838129832720:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.596382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:05.596421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518838129832720:2049] [node 2] ICP22 created new session: [3:7571518842424800112:2049]
2025-11-11T17:20:05.596457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518842424800112:2049] [node 2] ICS09 handshake done sender: [3:7571518842424800099:2117] self: [3:7571518842424800104:0] peer: [2:7571518842858030655:0] socket: 434 qp: -1
2025-11-11T17:20:05.596485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518842424800112:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:05.596533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:05.608340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:05.613894Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800086:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.613927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518842424800086:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:05.614500Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800106:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.614687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518842424800107:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.629275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518842424800079:2072] [node 4] ICP22 created new session: [3:7571518842424800114:2072]
2025-11-11T17:20:05.629274Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518842424800086:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:05.645735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518842424800114:2072] [node 4] ICS09 handshake done sender: [3:7571518842424800084:2111] self: [3:7571518842424800083:0] peer: [4:7571518841321316416:0] socket: 421 qp: -1
2025-11-11T17:20:05.645784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518842424800114:2072] [node 4] ICS10 traffic start
2025-11-11T17:20:05.650075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518842424800082:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:05.685723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:05.706237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:05.706297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518842424800082:2073] [node 6] ICP22 created new session: [3:7571518842424800118:2073]
2025-11-11T17:20:05.706356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518842424800118:2073] [node 6] ICS09 handshake done sender: [3:7571518842424800086:2112] self: [3:7571518842424800085:0] peer: [6:7571518843224238983:0] socket: 424 qp: -1
2025-11-11T17:20:05.706397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518842424800118:2073] [node 6] ICS10 traffic start
2025-11-11T17:20:05.706441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:05.722215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518842424800079:2072] [node 4] ICP35 event processing took too much time PendingConnection:278(153966507ns) {HandleHandshakeStatus:329(153963280ns) {DropIncomingHandshake:441(591ns) DropOutgoingHandshake:456(814ns) SwitchToState:178(650ns) ProcessPendingSessionEvents:495(123460ns) {ForwardSessionEventToSession:612(75176ns) ForwardSessionEventToSession:612(3543ns) ForwardSessionEventToSession:612(2731ns) ForwardSessionEventToSession:612(5143ns) ForwardSessionEventToSession:612(3402ns) ForwardSessionEventToSession:612(2329ns) ForwardSessionEventToSession:612(10050ns) ForwardSessionEventToSession:612(2893ns) ForwardSessionEventToSession:612(4864ns) ForwardSessionEventToSession:612(3001ns)}}}
2025-11-11T17:20:05.722303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518842424800079:2072] [node 4] ICP18 handshake (actor [3:7571518842424800107:2120]) is too old
2025-11-11T17:20:05.758044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518842424800082:2073] [node 6] ICP35 event processing took too much time PendingConnection:278(75740139ns) {HandleHandshakeStatus:329(75736700ns) {DropIncomingHandshake:441(13720ns) DropOutgoingHandshake:456(286ns) SwitchToState:178(606ns) ProcessPendingSessionEvents:495(19151550ns) {ForwardSessionEventToSession:612(19067545ns) ForwardSessionEventToSession:612(25797ns) ForwardSessionEventToSession:612(6854ns) ForwardSessionEventToSession:612(4558ns) ForwardSessionEventToSession:612(7033ns) ForwardSessionEventToSession:612(2400ns) ForwardSessionEventToSession:612(8914ns) ForwardSessionEventToSession:612(2344ns) ForwardSessionEventToSession:612(5087ns) ForwardSessionEventToSession:612(4665ns)}}}
2025-11-11T17:20:05.746169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518841022853092:2103] [node 4] ICS09 handshake done sender: [1:7571518841022853088:2118] self: [1:7571518841022853084:0] peer: [4:7571518841321316420:0] socket: 449 qp: -1
2025-11-11T17:20:05.753772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518841022853092:2103] [node 4] ICS10 traffic start
2025-11-11T17:20:05.769877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518839699060613:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(43980078ns) {HandleHandshakeStatus:395(31860177ns) {LogHandshakeFail:472(19818ns) {UpdateErrorStateLog:544(6473ns)} DropOutgoingHandshake:456(1637ns) ProcessPendingSessionEvents:495(7909692ns) {DropSessionEvent:512(20804ns) DropSessionEvent:512(5499ns) DropSessionEvent:512(4774ns) DropSessionEvent:512(4101ns) DropSessionEvent:512(27250ns) DropSessionEvent:512(10953ns) DropSessionEvent:512(4252ns) DropSessionEvent:512(3702ns) DropSessionEvent:512(4514ns) DropSessionEvent:512(5133ns)}}}
2025-11-11T17:20:05.770008Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518843994027982:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:05.381696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518843224238971:2062] [node 5] ICS10 traffic start
2025-11-11T17:20:05.381729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:05.773690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:05.774218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518841022853069:2103] [node 4] ICP35 event processing took too much time PendingConnection:278(121341281ns) {HandleHandshakeStatus:329(121338839ns) {DropIncomingHandshake:441(42719ns) DropOutgoingHandshake:456(551ns) SwitchToState:178(389ns) ProcessPendingSessionEvents:495(106321ns) {ForwardSessionEventToSession:612(60810ns) ForwardSessionEventToSession:612(3091ns) ForwardSessionEventToSession:612(6274ns) ForwardSessionEventToSession:612(3161ns) ForwardSessionEventToSession:612(2393ns) ForwardSessionEventToSession:612(5909ns) ForwardSessionEventToSession:612(5426ns) ForwardSessionEventToSession:612(4602ns) ForwardSessionEventToSession:612(2413ns) ForwardSessionEventToSession:612(3120ns)}}}
2025-11-11T17:20:05.777985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518839699060613:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.381956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518843224238912:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.428621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238975:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.429498Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518843224238965:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:05.429794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518843224238912:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.460716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238978:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.470194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518838929271592:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.473750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:05.473808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518843224238912:2067] [node 4] ICP22 created new session: [6:7571518843224238980:2067]
2025-11-11T17:20:05.473888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518843224238980:2067] [node 4] ICS09 handshake done sender: [6:7571518843224238965:2115] self: [6:7571518843224238968:0] peer: [4:7571518841321316397:0] socket: 413 qp: -1
2025-11-11T17:20:05.473927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518843224238980:2067] [node 4] ICS10 traffic start
2025-11-11T17:20:05.473964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:05.474513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518843224238912:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(44471958ns) {HandleHandshakeStatus:329(44470055ns) {DropIncomingHandshake:441(1296ns) DropOutgoingHandshake:456(5139ns) SwitchToState:178(486ns) ProcessPendingSessionEvents:495(97073ns) {ForwardSessionEventToSession:612(52395ns) ForwardSessionEventToSession:612(2928ns) ForwardSessionEventToSession:612(8728ns) ForwardSessionEventToSession:612(2892ns) ForwardSessionEventToSession:612(2378ns) ForwardSessionEventToSession:612(4963ns) ForwardSessionEventToSession:612(3242ns) ForwardSessionEventToSession:612(4927ns) ForwardSessionEventToSession:612(5244ns) ForwardSessionEventToSession:612(2058ns)}}}
2025-11-11T17:20:05.510155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238982:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.511440Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518843224238967:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.565652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518838929271594:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.597843Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518843224238978:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:05.598360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518838929271593:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:05.599472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518838929271608:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(92145814ns) {HandleHandshakeStatus:329(92143754ns) {DropIncomingHandshake:441(518ns) DropOutgoingHandshake:456(3790ns) SwitchToState:178(194ns) ProcessPendingSessionEvents:495(91783206ns) {ForwardSessionEventToSession:612(19981597ns) ForwardSessionEventToSession:612(8193465ns) ForwardSessionEventToSession:612(5219ns) ForwardSessionEventToSession:612(3583ns) ForwardSessionEventToSession:612(23766897ns) ForwardSessionEventToSession:612(6475ns) ForwardSessionEventToSession:612(15816404ns) ForwardSessionEventToSession:612(15987893ns) ForwardSessionEventToSession:612(3960ns) ForwardSessionEventToSession:612(7986939ns)}}}
2025-11-11T17:20:05.602535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518838929271594:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(42751561ns) {IncomingHandshake:261(42749705ns) {DropIncomingHandshake:441(257ns) PrepareNewSessionHandshake:476(1117ns) {SwitchToState:178(405ns)} GenerateSessionVirtualId:988(464ns) IssueIncomingHandshakeReply:193(28022033ns) {SwitchToState:178(3572ns)}}}
2025-11-11T17:20:05.610545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518838929271592:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(456473035ns) {Configure:104(456471595ns) {ProcessConfigured:135(56696581ns) {StartInitialHandshake:157(20842138ns) {DropHandshakes:469(3266ns) {DropIncomingHandshake:441(862ns) DropOutgoingHandshake:456(674ns)} GenerateSessionVirtualId:988(5535ns) PrepareNewSessionHandshake:476(2079ns) {SwitchToState:178(424ns)}} IncomingHandshake:261(35822119ns) {DropIncomingHandshake:441(352ns) PrepareNewSessionHandshake:476(658ns) {SwitchToState:178(296ns)} GenerateSessionVirtualId:988(663ns) IssueIncomingHandshakeReply:193(22770924ns) {SwitchToState:178(2135ns)}}}}}
2025-11-11T17:20:05.635015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238985:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.636568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518843224238986:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64817
2025-11-11T17:20:05.636637Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518843224238967:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:05.643975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518838929271612:2064] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:05.645667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518838929271594:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.674485Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:05.675557Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:05.676765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518838929271592:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.676849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:05.676896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518838929271592:2048] [node 1] ICP22 created new session: [6:7571518843224238987:2048]
2025-11-11T17:20:05.676956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518843224238987:2048] [node 1] ICS09 handshake done sender: [6:7571518838929271612:2064] self: [6:7571518843224238977:0] peer: [1:7571518836727885758:0] socket: 387 qp: -1
2025-11-11T17:20:05.676997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518843224238987:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:05.803071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:05.803139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518839699060613:2067] [node 4] ICP22 created new session: [5:7571518843994027989:2067]
2025-11-11T17:20:05.803193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518843994027989:2067] [node 4] ICS09 handshake done sender: [5:7571518843994027982:2122] self: [5:7571518843994027987:0] peer: [4:7571518841321316403:0] socket: 442 qp: -1
2025-11-11T17:20:05.803240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518843994027989:2067] [node 4] ICS10 traffic start
2025-11-11T17:20:05.803284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:05.803543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518839699060613:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(25514150ns) {HandleHandshakeStatus:329(25510349ns) {DropIncomingHandshake:441(1344ns) DropOutgoingHandshake:456(896ns) SwitchToState:178(799ns) ProcessPendingSessionEvents:495(536ns)}}
2025-11-11T17:20:05.677055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:05.685946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518838929271592:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:05.686474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518838929271593:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518843224238978:2121] held: no
2025-11-11T17:20:05.686710Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518843224238978:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:05.687460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518838929271593:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:05.687541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:05.687602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518838929271593:2049] [node 2] ICP22 created new session: [6:7571518843224238989:2049]
2025-11-11T17:20:05.687646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518843224238989:2049] [node 2] ICS09 handshake done sender: [6:7571518843224238978:2121] self: [6:7571518843224238984:0] peer: [2:7571518842858030672:0] socket: 398 qp: -1
2025-11-11T17:20:05.687677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518843224238989:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:05.687726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:05.692722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:05.702115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518838929271594:2050] [node 3] ICP22 created new session: [6:7571518843224238991:2050]
2025-11-11T17:20:05.709881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518843224238991:2050] [node 3] ICS09 handshake done sender: [6:7571518843224238967:2117] self: [6:7571518843224238983:0] peer: [3:7571518842424800085:0] socket: 425 qp: -1
2025-11-11T17:20:05.709929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518843224238991:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:05.710007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:05.716350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518838929271594:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(64664576ns) {HandleHandshakeStatus:329(64660570ns) {DropIncomingHandshake:441(1119ns) DropOutgoingHandshake:456(14077ns) SwitchToState:178(667ns) ProcessPendingSessionEvents:495(113852ns) {ForwardSessionEventToSession:612(61461ns) ForwardSessionEventToSession:612(4244ns) ForwardSessionEventToSession:612(6754ns) ForwardSessionEventToSession:612(3133ns) ForwardSessionEventToSession:612(7819ns) ForwardSessionEventToSession:612(3692ns) ForwardSessionEventToSession:612(3529ns) ForwardSessionEventToSession:612(6786ns) ForwardSessionEventToSession:612(5072ns) ForwardSessionEventToSession:612(3093ns)}}}
2025-11-11T17:20:05.880738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:05.880988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518842858030678:2048] [node 1] ICS01 socket: 402 reason# EndOfStream
2025-11-11T17:20:05.881033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518838563063294:2048] [node 1] ICP30 unregister session Session# [2:7571518842858030678:2048] VirtualId# [2:7571518842858030675:0]
2025-11-11T17:20:05.881068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518842858030678:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:05.881173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:05.881219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:05.883263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:05.883534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518842424800109:2048] [node 1] ICS01 socket: 417 reason# EndOfStream
2025-11-11T17:20:05.883566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518838129832719:2048] [node 1] ICP30 unregister session Session# [3:7571518842424800109:2048] VirtualId# [3:7571518842424800097:0]
2025-11-11T17:20:05.883616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518842424800109:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:05.883698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:05.883736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:05.897871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:05.898203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518843994027961:2048] [node 1] ICS07 socket disconnect 390 reason# EndOfStream
2025-11-11T17:20:05.898236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518843994027961:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:05.898361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:05.898416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518843994027961:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:05.899586Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address
2025-11-11T17:20:05.899901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518839699060590:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:05.899952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518843994027961:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:05.899981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518839699060590:2048] [node 1] ICP30 unregister session Session# [5:7571518843994027961:2048] VirtualId# [5:7571518843994027919:0]
2025-11-11T17:20:05.900032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:05.900071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518839699060590:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:05.900090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:05.897873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:05.898155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518843224238987:2048] [node 1] ICS07 socket disconnect 387 reason# EndOfStream
2025-11-11T17:20:05.898189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518843224238987:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:05.898259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:05.898295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518843224238987:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:05.906324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518839699060590:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:05.905754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:05.906003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518841321316422:2048] [node 1] ICS01 socket: 452 reason# EndOfStream
2025-11-11T17:20:05.906049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518837026349028:2048] [node 1] ICP30 unregister session Session# [4:7571518841321316422:2048] VirtualId# [4:7571518841321316420:0]
2025-11-11T17:20:05.906081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518841321316422:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:05.906185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:05.906237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:05.909776Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address
2025-11-11T17:20:05.913722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518838929271592:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:05.913789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518843224238987:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:05.913831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518838929271592:2048] [node 1] ICP30 unregister session Session# [6:7571518843224238987:2048] VirtualId# [6:7571518843224238977:0]
2025-11-11T17:20:05.913896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:05.913927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518838929271592:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:05.913947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:05.935487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518838929271592:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:06.077078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:06.077325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518842424800112:2049] [node 2] ICS01 socket: 434 reason# EndOfStream
2025-11-11T17:20:06.077369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518838129832720:2049] [node 2] ICP30 unregister session Session# [3:7571518842424800112:2049] VirtualId# [3:7571518842424800104:0]
2025-11-11T17:20:06.077403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518842424800112:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.077470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:06.077509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:06.089799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:06.090042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518841321316408:2049] [node 2] ICS01 socket: 410 reason# EndOfStream
2025-11-11T17:20:06.090088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518837026349029:2049] [node 2] ICP30 unregister session Session# [4:7571518841321316408:2049] VirtualId# [4:7571518841321316406:0]
2025-11-11T17:20:06.090120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518841321316408:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.090218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:06.090261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:06.090309Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:06.090523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518843224238989:2049] [node 2] ICS01 socket: 398 reason# EndOfStream
2025-11-11T17:20:06.090553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518838929271593:2049] [node 2] ICP30 unregister session Session# [6:7571518843224238989:2049] VirtualId# [6:7571518843224238984:0]
2025-11-11T17:20:06.090581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518843224238989:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.090664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:06.090704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:06.093786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:06.094142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518843994027983:2049] [node 2] ICS07 socket disconnect 411 reason# EndOfStream
2025-11-11T17:20:06.094190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518843994027983:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.094361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:06.094428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518843994027983:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:06.137717Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23977) Couldn't connect to any resolved address
2025-11-11T17:20:06.138105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518839699060591:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23977) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:06.138160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518843994027983:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:06.138192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518839699060591:2049] [node 2] ICP30 unregister session Session# [5:7571518843994027983:2049] VirtualId# [5:7571518843994027975:0]
2025-11-11T17:20:06.138237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:06.138271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518839699060591:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:06.138286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:06.150144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518839699060591:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:06.208753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:06.209008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518843224238991:2050] [node 3] ICS01 socket: 425 reason# EndOfStream
2025-11-11T17:20:06.209048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518838929271594:2050] [node 3] ICP30 unregister session Session# [6:7571518843224238991:2050] VirtualId# [6:7571518843224238983:0]
2025-11-11T17:20:06.209083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518843224238991:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.209197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:06.209243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:06.209879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:06.210126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518843994027985:2050] [node 3] ICS01 socket: 423 reason# EndOfStream
2025-11-11T17:20:06.210159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518839699060592:2050] [node 3] ICP30 unregister session Session# [5:7571518843994027985:2050] VirtualId# [5:7571518843994027977:0]
2025-11-11T17:20:06.210189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518843994027985:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.210265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:06.210303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:06.210387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:06.210584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518841321316424:2050] [node 3] ICS01 socket: 422 reason# EndOfStream
2025-11-11T17:20:06.210608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518837026349030:2050] [node 3] ICP30 unregister session Session# [4:7571518841321316424:2050] VirtualId# [4:7571518841321316416:0]
2025-11-11T17:20:06.210634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518841321316424:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.210675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:06.210703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:06.367322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:06.367609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518843994027989:2067] [node 4] ICS01 socket: 442 reason# EndOfStream
2025-11-11T17:20:06.367657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518839699060613:2067] [node 4] ICP30 unregister session Session# [5:7571518843994027989:2067] VirtualId# [5:7571518843994027987:0]
2025-11-11T17:20:06.367688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518843994027989:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.367773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:06.367823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:06.369809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:06.370180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518843224238980:2067] [node 4] ICS01 socket: 413 reason# EndOfStream
2025-11-11T17:20:06.370246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518843224238912:2067] [node 4] ICP30 unregister session Session# [6:7571518843224238980:2067] VirtualId# [6:7571518843224238968:0]
2025-11-11T17:20:06.370337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518843224238980:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.370492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:06.370571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:06.425761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:06.426027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518843224238971:2062] [node 5] ICS01 socket: 419 reason# EndOfStream
2025-11-11T17:20:06.426068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518838929271608:2062] [node 5] ICP30 unregister session Session# [6:7571518843224238971:2062] VirtualId# [6:7571518843224238969:0]
2025-11-11T17:20:06.426103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518843224238971:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:06.426204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:06.426244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:06.653053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518847393550642:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:06.653109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518847393550643:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:06.725914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518845129107512:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:06.725959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518845129107513:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:06.805891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518846712033385:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:06.805936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518846712033386:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:06.845849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518846522220763:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:06.845890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518846522220764:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:06.847025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518846522220762:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:06.917099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518846340172772:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:06.917143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518846340172773:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:06.917170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518846340172774:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:07.038899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518848100643994:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:07.038939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518848100643995:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:07.038968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518848100643996:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:07.246271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518851007000710:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.252843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518850817188085:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-11T17:20:07.207031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518851688517985:2093] [node 5] ICP01 ready to work
2025-11-11T17:20:07.286625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518852395611306:2061] [node 5] ICP01 ready to work
2025-11-11T17:20:07.409750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518851007000728:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.437697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518849424074870:2072] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.449914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518851007000751:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23395
2025-11-11T17:20:07.463523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518846522220762:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(177562229ns) {Configure:104(177557062ns) {ProcessConfigured:135(67041ns) {StartInitialHandshake:157(64099ns) {DropHandshakes:469(2476ns) {DropIncomingHandshake:441(572ns) DropOutgoingHandshake:456(431ns)} GenerateSessionVirtualId:988(723ns) PrepareNewSessionHandshake:476(5114ns) {SwitchToState:178(585ns)}}}}}
2025-11-11T17:20:07.435981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518847393550643:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(82124746ns) {Configure:104(82119595ns) {ProcessConfigured:135(78824087ns) {StartInitialHandshake:157(78822131ns) {DropHandshakes:469(2384ns) {DropIncomingHandshake:441(595ns) DropOutgoingHandshake:456(457ns)} GenerateSessionVirtualId:988(578ns) PrepareNewSessionHandshake:476(3065ns) {SwitchToState:178(1353ns)}}}}}
2025-11-11T17:20:07.504630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518851007000753:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23395
2025-11-11T17:20:07.514253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518849424074872:2070] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.437630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518851688518004:2094] [node 4] ICP01 ready to work
2025-11-11T17:20:07.502183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518850817188129:2068] [node 5] ICP01 ready to work
2025-11-11T17:20:07.511622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518846340172774:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(201685038ns) {Configure:104(201681908ns) {ProcessConfigured:135(77691ns) {StartInitialHandshake:157(75093ns) {DropHandshakes:469(2200ns) {DropIncomingHandshake:441(572ns) DropOutgoingHandshake:456(466ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:476(3877ns) {SwitchToState:178(739ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.564864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518852395611363:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:07.565080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518852395611361:2065] [node 4] ICP01 ready to work
2025-11-11T17:20:07.509687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518851688518006:2098] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.538229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518851007000755:2075] [node 6] ICP01 ready to work
2025-11-11T17:20:07.512315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140133:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.497790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518850817188125:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:07.551578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518849424074873:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-11T17:20:07.600142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518847393550642:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(382184834ns) {Configure:104(382179994ns) {ProcessConfigured:135(58325ns) {StartInitialHandshake:157(55348ns) {DropHandshakes:469(1984ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(315ns)} GenerateSessionVirtualId:988(646ns) PrepareNewSessionHandshake:476(1192ns) {SwitchToState:178(423ns)}}}}}
2025-11-11T17:20:07.564006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518851007000756:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23395
2025-11-11T17:20:07.565786Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518850817188085:2074] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.512484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140134:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.512533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140135:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.541687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518850635140137:2066] [node 6] ICP01 ready to work
2025-11-11T17:20:07.541641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518850635140138:2067] [node 4] ICP01 ready to work
2025-11-11T17:20:07.590201Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518850635140133:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.625741Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518850635140134:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.646063Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518850635140135:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.646383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518846340172774:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.646452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518850635140138:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.646540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140144:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.541672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518849424074874:2074] [node 6] ICP01 ready to work
2025-11-11T17:20:07.586616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518846712033386:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(277042350ns) {Configure:104(277034789ns) {ProcessConfigured:135(69512ns) {StartInitialHandshake:157(66015ns) {DropHandshakes:469(2324ns) {DropIncomingHandshake:441(697ns) DropOutgoingHandshake:456(315ns)} GenerateSessionVirtualId:988(1067ns) PrepareNewSessionHandshake:476(1853ns) {SwitchToState:178(924ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.586939Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518851007000756:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.603050Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074873:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.578570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518850817188139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-11T17:20:07.654370Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518852395611363:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.655661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518846340172772:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(340007953ns) {Configure:104(339997728ns) {ProcessConfigured:135(45252865ns) {StartInitialHandshake:157(45249301ns) {DropHandshakes:469(2750ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(876ns)} GenerateSessionVirtualId:988(681ns) PrepareNewSessionHandshake:476(4058ns) {SwitchToState:178(1634ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.587408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518850817188138:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-11T17:20:07.660023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518851688518010:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16714
2025-11-11T17:20:07.588252Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518851007000751:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.608152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518849424074870:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(103225333ns) {Configure:104(103219707ns) {ProcessConfigured:135(10152641ns) {StartInitialHandshake:157(10150568ns) {DropHandshakes:469(2675ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(513ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:476(2306ns) {SwitchToState:178(766ns)}}}}}
2025-11-11T17:20:07.661221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518852395611361:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.660184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518851688518012:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16714
2025-11-11T17:20:07.662513Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518850635140144:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.605991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518846522220764:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(335548865ns) {Configure:104(311908869ns) {ProcessConfigured:135(85384ns) {StartInitialHandshake:157(82175ns) {DropHandshakes:469(857ns) {DropIncomingHandshake:441(128ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(1753ns) {SwitchToState:178(816ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.641785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518851007000712:2076] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.723251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518849424074885:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-11T17:20:07.718903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518848100643994:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(366914783ns) {Configure:104(366908317ns) {ProcessConfigured:135(11773941ns) {StartInitialHandshake:157(11771676ns) {DropHandshakes:469(2383ns) {DropIncomingHandshake:441(746ns) DropOutgoingHandshake:456(408ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:476(3027ns) {SwitchToState:178(1380ns)}}}}}
2025-11-11T17:20:07.618410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518846522220763:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(372505982ns) {Configure:104(372496394ns) {ProcessConfigured:135(61353ns) {StartInitialHandshake:157(58875ns) {DropHandshakes:469(1595ns) {DropIncomingHandshake:441(364ns) DropOutgoingHandshake:456(296ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:476(1707ns) {SwitchToState:178(878ns)}}}}}
2025-11-11T17:20:07.681935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140148:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.690796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518851688518006:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(148939745ns) {Configure:104(148930303ns) {ProcessConfigured:135(55908ns) {StartInitialHandshake:157(52877ns) {DropHandshakes:469(1585ns) {DropIncomingHandshake:441(315ns) DropOutgoingHandshake:456(248ns)} GenerateSessionVirtualId:988(1170ns) PrepareNewSessionHandshake:476(1319ns) {SwitchToState:178(417ns)}}}}}
2025-11-11T17:20:07.701407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518851688517985:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(435448508ns) {Configure:104(427080871ns) {ProcessConfigured:135(15177617ns) {StartInitialHandshake:157(15173889ns) {DropHandshakes:469(2713ns) {DropIncomingHandshake:441(825ns) DropOutgoingHandshake:456(436ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:476(4247ns) {SwitchToState:178(912ns)}}}}}
2025-11-11T17:20:07.706234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518851688518018:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16714
2025-11-11T17:20:07.722407Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518010:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.726399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518851688518004:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(228515520ns) {Configure:104(228506570ns) {ProcessConfigured:135(64633ns) {StartInitialHandshake:157(62876ns) {DropHandshakes:469(1962ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(361ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:476(1377ns) {SwitchToState:178(466ns)}}}}}
2025-11-11T17:20:07.738609Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518012:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.741785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518851688518009:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16714
2025-11-11T17:20:07.757136Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518009:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.765699Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688517998:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.765740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518851688517998:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.766172Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518018:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.749738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518850635140135:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:07.791134Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518851688517998:2107] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:07.737000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074876:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.641811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518851007000712:2076] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.766182Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:07.645791Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518851007000712:2076] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:07.644592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518850817188143:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-11T17:20:07.737024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518849424074876:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.754438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518849424074889:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-11T17:20:07.754707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518845129107513:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(355806920ns) {Configure:104(355801962ns) {ProcessConfigured:135(15783428ns) {StartInitialHandshake:157(15780817ns) {DropHandshakes:469(2532ns) {DropIncomingHandshake:441(398ns) DropOutgoingHandshake:456(621ns)} GenerateSessionVirtualId:988(432ns) PrepareNewSessionHandshake:476(2905ns) {SwitchToState:178(352ns)}}}}}
2025-11-11T17:20:07.765818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518845129107512:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(382857271ns) {Configure:104(382853416ns) {ProcessConfigured:135(42973710ns) {StartInitialHandshake:157(42971358ns) {DropHandshakes:469(7945996ns) {DropIncomingHandshake:441(438ns) DropOutgoingHandshake:456(3347ns)} GenerateSessionVirtualId:988(2804ns) PrepareNewSessionHandshake:476(2694ns) {SwitchToState:178(1077ns)}}}}}
2025-11-11T17:20:07.769431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518849424074887:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-11T17:20:07.769661Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518849424074876:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:07.785834Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074885:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.822039Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:07.822519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074887:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.822578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518849424074872:2070] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:07.829782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518849424074872:2070] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.809814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518852395611371:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:07.811131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518847393550643:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(16299597ns) {IncomingHandshake:261(16297918ns) {DropIncomingHandshake:441(286ns) PrepareNewSessionHandshake:476(1086ns) {SwitchToState:178(571ns)} GenerateSessionVirtualId:988(300ns) IssueIncomingHandshakeReply:193(7928ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.657713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518846522220764:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.806018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518850635140137:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(210347169ns) {Configure:104(210341841ns) {ProcessConfigured:135(11843110ns) {StartInitialHandshake:157(11839854ns) {DropHandshakes:469(2324ns) {DropIncomingHandshake:441(586ns) DropOutgoingHandshake:456(377ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:476(7744ns) {SwitchToState:178(1505ns)}}}}}
2025-11-11T17:20:07.657679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518846712033385:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(339962699ns) {Configure:104(339958581ns) {ProcessConfigured:135(33817591ns) {StartInitialHandshake:157(33814511ns) {DropHandshakes:469(2834ns) {DropIncomingHandshake:441(555ns) DropOutgoingHandshake:456(501ns)} GenerateSessionVirtualId:988(725ns) PrepareNewSessionHandshake:476(3075ns) {SwitchToState:178(1199ns)}}}}}
2025-11-11T17:20:07.853734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518845129107512:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.830058Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518852395611363:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:07.811215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518847393550643:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:07.673921Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518851007000745:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.673956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518851007000745:2106] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.686249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518851007000710:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:07.703459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518851007000763:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23395
2025-11-11T17:20:07.706689Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518851007000753:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.730096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518846712033385:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.730730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:07.741837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518851007000745:2106] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:07.742122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518851007000728:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:07.749853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:07.749906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518851007000728:2069] [node 5] ICP22 created new session: [3:7571518851007000768:2069]
2025-11-11T17:20:07.749952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518851007000768:2069] [node 5] ICS09 handshake done sender: [3:7571518851007000745:2106] self: [3:7571518851007000744:0] peer: [5:7571518850635140145:0] socket: 393 qp: -1
2025-11-11T17:20:07.749997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518851007000768:2069] [node 5] ICS10 traffic start
2025-11-11T17:20:07.750066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:07.765925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518851007000710:2068] [node 4] ICP22 created new session: [3:7571518851007000767:2068]
2025-11-11T17:20:07.767346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518851007000767:2068] [node 4] ICS09 handshake done sender: [3:7571518851007000712:2076] self: [3:7571518851007000711:0] peer: [4:7571518850817188142:0] socket: 390 qp: -1
2025-11-11T17:20:07.773823Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:20:07.785697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518851007000767:2068] [node 4] ICS10 traffic start
2025-11-11T17:20:07.798221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:07.842057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518851007000771:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23395
2025-11-11T17:20:07.854892Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518851007000753:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:07.870179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518846712033385:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:07.870246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:07.870295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518846712033385:2048] [node 1] ICP22 created new session: [3:7571518851007000772:2048]
2025-11-11T17:20:07.870336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518851007000772:2048] [node 1] ICS09 handshake done sender: [3:7571518851007000753:2113] self: [3:7571518851007000765:0] peer: [1:7571518851688517988:0] socket: 399 qp: -1
2025-11-11T17:20:07.870369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518851007000772:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:07.870419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:07.870821Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:07.873761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518846712033385:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:07.878184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518851007000710:2068] [node 4] ICP35 event processing took too much time PendingConnection:278(203727805ns) {HandleHandshakeStatus:329(203725329ns) {DropIncomingHandshake:441(861ns) DropOutgoingHandshake:456(1070ns) SwitchToState:178(381ns) ProcessPendingSessionEvents:495(110284ns) {ForwardSessionEventToSession:612(60505ns) ForwardSessionEventToSession:612(4032ns) ForwardSessionEventToSession:612(5782ns) ForwardSessionEventToSession:612(8515ns) ForwardSessionEventToSession:612(3016ns) ForwardSessionEventToSession:612(5346ns) ForwardSessionEventToSession:612(2389ns) ForwardSessionEventToSession:612(1925ns) ForwardSessionEventToSession:612(4534ns) ForwardSessionEventToSession:612(3189ns)}}}
2025-11-11T17:20:07.878851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518851007000774:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23395
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.806117Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:07.885676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:07.885745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074879:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.885777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518849424074879:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.887241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518845129107512:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(42636561ns) {IncomingHandshake:261(19895110ns) {DropIncomingHandshake:441(181ns) PrepareNewSessionHandshake:476(8671764ns) {SwitchToState:178(3050ns)} GenerateSessionVirtualId:988(742ns) IssueIncomingHandshakeReply:193(68721ns) {SwitchToState:178(514ns)}}}
2025-11-11T17:20:07.891552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518849424074894:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-11T17:20:07.892999Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518849424074879:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:07.893233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518849424074870:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:07.893281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:07.893322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518849424074870:2072] [node 5] ICP22 created new session: [2:7571518849424074896:2072]
2025-11-11T17:20:07.893362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518849424074896:2072] [node 5] ICS09 handshake done sender: [2:7571518849424074879:2112] self: [2:7571518849424074878:0] peer: [5:7571518850635140158:0] socket: 415 qp: -1
2025-11-11T17:20:07.893389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518849424074896:2072] [node 5] ICS10 traffic start
2025-11-11T17:20:07.893445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:07.897644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518849424074872:2070] [node 4] ICP22 created new session: [2:7571518849424074893:2070]
2025-11-11T17:20:07.902032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518849424074893:2070] [node 4] ICS09 handshake done sender: [2:7571518849424074876:2111] self: [2:7571518849424074875:0] peer: [4:7571518850817188153:0] socket: 408 qp: -1
2025-11-11T17:20:07.657858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518846522220764:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(32035036ns) {IncomingHandshake:261(32033022ns) {DropIncomingHandshake:441(361ns) PrepareNewSessionHandshake:476(1593ns) {SwitchToState:178(681ns)} GenerateSessionVirtualId:988(553ns) IssueIncomingHandshakeReply:193(20032809ns) {SwitchToState:178(1457ns)}}}
2025-11-11T17:20:07.851026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518852395611361:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:07.829862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518851688518004:2094] [node 4] ICP35 event processing took too much time PendingConnection:278(27859260ns) {IncomingHandshake:261(27856718ns) {DropIncomingHandshake:441(1864ns) PrepareNewSessionHandshake:476(3659ns) {SwitchToState:178(2754ns)} GenerateSessionVirtualId:988(593ns) IssueIncomingHandshakeReply:193(10087ns)}}
2025-11-11T17:20:07.918058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518851007000755:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(351612712ns) {Configure:104(343406026ns) {ProcessConfigured:135(15850665ns) {StartInitialHandshake:157(15847439ns) {DropHandshakes:469(2433ns) {DropIncomingHandshake:441(561ns) DropOutgoingHandshake:456(452ns)} GenerateSessionVirtualId:988(917ns) PrepareNewSessionHandshake:476(4221ns) {SwitchToState:178(2176ns)}}}}}
2025-11-11T17:20:07.866073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518852395611372:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.902740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:07.660584Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518850817188131:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.810163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518850635140138:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518850635140135:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.876129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:07.925707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518849424074893:2070] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.810250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518850635140138:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:07.946737Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518851007000774:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.660628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518850817188131:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.671334Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518850817188131:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:07.671575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518850817188129:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:07.671635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.942805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.910293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518852395611361:2065] [node 4] ICP22 created new session: [6:7571518852395611373:2065]
2025-11-11T17:20:07.887098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518847393550643:2049] [node 3] ICP22 created new session: [1:7571518851688518027:2049]
2025-11-11T17:20:07.936524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518852395611373:2065] [node 4] ICS09 handshake done sender: [6:7571518852395611363:2112] self: [6:7571518852395611367:0] peer: [4:7571518850817188133:0] socket: 409 qp: -1
2025-11-11T17:20:07.946500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518852395611370:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:07.949669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518852395611373:2065] [node 4] ICS10 traffic start
2025-11-11T17:20:07.953961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518852395611375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:07.969730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:07.982526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518852395611377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:07.986376Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518852395611370:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.989711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518852395611372:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.990139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518852395611361:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(153250597ns) {HandleHandshakeStatus:329(153247534ns) {DropIncomingHandshake:441(1569ns) DropOutgoingHandshake:456(17046ns) SwitchToState:178(4689ns) ProcessPendingSessionEvents:495(119792ns) {ForwardSessionEventToSession:612(71412ns) ForwardSessionEventToSession:612(3261ns) ForwardSessionEventToSession:612(8425ns) ForwardSessionEventToSession:612(3140ns) ForwardSessionEventToSession:612(2901ns) ForwardSessionEventToSession:612(8094ns) ForwardSessionEventToSession:612(3763ns) ForwardSessionEventToSession:612(2647ns) ForwardSessionEventToSession:612(5530ns) ForwardSessionEventToSession:612(2489ns)}}}
2025-11-11T17:20:07.998243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518852395611306:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.999476Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518852395611375:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.810291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:07.966290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518846712033386:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.671680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518850817188129:2068] [node 5] ICP22 created new session: [4:7571518850817188145:2068]
2025-11-11T17:20:07.671721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518850817188145:2068] [node 5] ICS09 handshake done sender: [4:7571518850817188131:2115] self: [4:7571518850817188130:0] peer: [5:7571518850635140146:0] socket: 401 qp: -1
2025-11-11T17:20:07.950684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518849424074870:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(41505898ns) {HandleHandshakeStatus:329(41503592ns) {DropIncomingHandshake:441(394ns) DropOutgoingHandshake:456(772ns) SwitchToState:178(595ns) ProcessPendingSessionEvents:495(41160404ns) {ForwardSessionEventToSession:612(60939ns) ForwardSessionEventToSession:612(3008ns) ForwardSessionEventToSession:612(20000609ns) ForwardSessionEventToSession:612(23398ns) ForwardSessionEventToSession:612(4156ns) ForwardSessionEventToSession:612(11990216ns) ForwardSessionEventToSession:612(7989381ns) ForwardSessionEventToSession:612(1048318ns) ForwardSessionEventToSession:612(5925ns) ForwardSessionEventToSession:612(2817ns)}}}
2025-11-11T17:20:08.021792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518848100643996:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.889637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518008:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.986753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518846712033386:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19434580ns) {IncomingHandshake:261(19432363ns) {DropIncomingHandshake:441(1011ns) PrepareNewSessionHandshake:476(3607ns) {SwitchToState:178(2169ns)} GenerateSessionVirtualId:988(755ns) IssueIncomingHandshakeReply:193(5979036ns) {SwitchToState:178(2295ns)}}}
2025-11-11T17:20:07.810353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518850635140138:2067] [node 4] ICP22 created new session: [5:7571518850635140154:2067]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:08.025767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518848100643994:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.671756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518850817188145:2068] [node 5] ICS10 traffic start
2025-11-11T17:20:07.889683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518851688518008:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.964599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518849424074874:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(350190063ns) {Configure:104(350182799ns) {ProcessConfigured:135(34534902ns) {StartInitialHandshake:157(34531772ns) {DropHandshakes:469(7740965ns) {DropIncomingHandshake:441(604ns) DropOutgoingHandshake:456(3005ns)} GenerateSessionVirtualId:988(2495ns) PrepareNewSessionHandshake:476(2880ns) {SwitchToState:178(1405ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:08.041654Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518851007000776:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.966077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074894:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.994460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518849424074870:2072] [node 5] ICP18 handshake (actor [2:7571518849424074894:2118]) is too old
2025-11-11T17:20:07.998077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518849424074903:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-11T17:20:07.999347Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074884:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.671799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.810394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518850635140154:2067] [node 4] ICS09 handshake done sender: [5:7571518850635140135:2113] self: [5:7571518850635140146:0] peer: [4:7571518850817188130:0] socket: 404 qp: -1
2025-11-11T17:20:07.999382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518849424074884:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:08.002176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518849424074872:2070] [node 4] ICP35 event processing took too much time PendingConnection:278(172219889ns) {HandleHandshakeStatus:329(172215970ns) {DropIncomingHandshake:441(186ns) DropOutgoingHandshake:456(626ns) SwitchToState:178(536ns) ProcessPendingSessionEvents:495(118516ns) {ForwardSessionEventToSession:612(68493ns) ForwardSessionEventToSession:612(3860ns) ForwardSessionEventToSession:612(5415ns) ForwardSessionEventToSession:612(3000ns) ForwardSessionEventToSession:612(5163ns) ForwardSessionEventToSession:612(3213ns) ForwardSessionEventToSession:612(3247ns) ForwardSessionEventToSession:612(4465ns) ForwardSessionEventToSession:612(4888ns) ForwardSessionEventToSession:612(2961ns)}}}
2025-11-11T17:20:08.031642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518849424074900:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:08.031664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518849424074900:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:08.032401Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518849424074885:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:08.032874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518849424074884:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:08.053784Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518849424074900:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:08.059291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518845129107513:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:08.061859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518845129107512:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.086387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518849424074874:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:08.086467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:07.893737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518851688518027:2049] [node 3] ICS09 handshake done sender: [1:7571518851688517998:2107] self: [1:7571518851688517988:0] peer: [3:7571518851007000765:0] socket: 397 qp: -1
2025-11-11T17:20:07.689089Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518850817188085:2074] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:07.696576Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518850817188134:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.696619Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518850817188134:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.697606Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518850817188134:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:07.704972Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518850817188138:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.717983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518846522220764:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:07.730496Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518850817188139:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.733896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:07.733958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518846522220764:2050] [node 3] ICP22 created new session: [4:7571518850817188149:2050]
2025-11-11T17:20:07.734010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518850817188149:2050] [node 3] ICS09 handshake done sender: [4:7571518850817188085:2074] self: [4:7571518850817188142:0] peer: [3:7571518851007000711:0] socket: 391 qp: -1
2025-11-11T17:20:07.734046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518850817188149:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:07.734086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:07.734559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518846522220764:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(16346495ns) {HandleHandshakeStatus:329(16343522ns) {DropIncomingHandshake:441(1126ns) DropOutgoingHandshake:456(6232ns) SwitchToState:178(482ns) ProcessPendingSessionEvents:495(115343ns) {ForwardSessionEventToSession:612(64879ns) ForwardSessionEventToSession:612(6532ns) ForwardSessionEventToSession:612(5951ns) ForwardSessionEventToSession:612(3711ns) ForwardSessionEventToSession:612(4905ns) ForwardSessionEventToSession:612(2761ns) ForwardSessionEventToSession:612(3923ns) ForwardSessionEventToSession:612(6223ns) ForwardSessionEventToSession:612(2548ns) ForwardSessionEventToSession:612(5331ns)}}}
2025-11-11T17:20:07.734664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518850817188125:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:07.734736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:07.734810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518850817188125:2067] [node 6] ICP22 created new session: [4:7571518850817188151:2067]
2025-11-11T17:20:07.734856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518850817188151:2067] [node 6] ICS09 handshake done sender: [4:7571518850817188134:2116] self: [4:7571518850817188133:0] peer: [6:7571518852395611367:0] socket: 407 qp: -1
2025-11-11T17:20:07.734882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518850817188151:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:07.734920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:07.735544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518846522220763:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.735829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518850817188129:2068] [node 5] ICP18 handshake (actor [4:7571518850817188139:2119]) is too old
2025-11-11T17:20:07.751136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518850817188155:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-11T17:20:07.778300Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518850817188138:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:07.802169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518846522220763:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:07.802270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:07.802327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518846522220763:2049] [node 2] ICP22 created new session: [4:7571518850817188156:2049]
2025-11-11T17:20:07.802395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518850817188156:2049] [node 2] ICS09 handshake done sender: [4:7571518850817188138:2118] self: [4:7571518850817188153:0] peer: [2:7571518849424074875:0] socket: 410 qp: -1
2025-11-11T17:20:07.802434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518850817188156:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:07.802506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:08.041700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518851007000776:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:08.086511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518849424074874:2074] [node 6] ICP22 created new session: [2:7571518853719042203:2074]
2025-11-11T17:20:08.086557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518853719042203:2074] [node 6] ICS09 handshake done sender: [2:7571518849424074900:2119] self: [2:7571518849424074899:0] peer: [6:7571518856690578677:0] socket: 449 qp: -1
2025-11-11T17:20:08.086589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518853719042203:2074] [node 6] ICS10 traffic start
2025-11-11T17:20:08.086640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:08.093631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:08.100629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518845129107512:2048] [node 1] ICP22 created new session: [2:7571518853719042202:2048]
2025-11-11T17:20:08.109753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:08.121852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518845129107513:2049] [node 3] ICP22 created new session: [2:7571518853719042205:2049]
2025-11-11T17:20:08.123446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518853719042202:2048] [node 1] ICS09 handshake done sender: [2:7571518849424074885:2115] self: [2:7571518849424074891:0] peer: [1:7571518851688518007:0] socket: 424 qp: -1
2025-11-11T17:20:08.125779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518853719042205:2049] [node 3] ICS09 handshake done sender: [2:7571518849424074884:2114] self: [2:7571518849424074882:0] peer: [3:7571518851007000779:0] socket: 405 qp: -1
2025-11-11T17:20:08.029840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518848100643996:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(20163324ns) {IncomingHandshake:261(20162158ns) {DropIncomingHandshake:441(230ns) PrepareNewSessionHandshake:476(1571ns) {SwitchToState:178(284ns)} GenerateSessionVirtualId:988(520ns) IssueIncomingHandshakeReply:193(12075205ns) {SwitchToState:178(1851ns)}}}
2025-11-11T17:20:08.030001Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518852395611377:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:08.030444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518848100643995:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:08.036602Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14477) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:08.045852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518852395611306:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14477) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518852395611372:2118] held: no
2025-11-11T17:20:08.058900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518848100643994:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(26695467ns) {IncomingHandshake:261(26693870ns) {DropIncomingHandshake:441(1375ns) PrepareNewSessionHandshake:476(2897ns) {SwitchToState:178(1852ns)} GenerateSessionVirtualId:988(893ns) IssueIncomingHandshakeReply:193(60429ns) {SwitchToState:178(281ns)}}}
2025-11-11T17:20:08.066048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518856690578678:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:08.066833Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518852395611372:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:08.067022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518852395611306:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.067085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:08.067136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518852395611306:2061] [node 5] ICP22 created new session: [6:7571518856690578680:2061]
2025-11-11T17:20:08.067187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518856690578680:2061] [node 5] ICS09 handshake done sender: [6:7571518852395611372:2118] self: [6:7571518852395611378:0] peer: [5:7571518850635140152:0] socket: 441 qp: -1
2025-11-11T17:20:08.067219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518856690578680:2061] [node 5] ICS10 traffic start
2025-11-11T17:20:08.067253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:08.100349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518856690578682:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:08.102243Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:08.103326Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:08.104205Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:08.105123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518848100643994:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518852395611370:2116] held: no
2025-11-11T17:20:08.105220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518848100643995:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518852395611377:2120] held: no
2025-11-11T17:20:08.105314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518848100643996:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518852395611375:2119] held: no
2025-11-11T17:20:08.107176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518852395611375:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:08.110144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518856690578683:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:08.118740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518848100643996:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.810426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518850635140154:2067] [node 4] ICS10 traffic start
2025-11-11T17:20:07.810479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:07.813657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518846340172773:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(428161864ns) {Configure:104(428157670ns) {ProcessConfigured:135(28147904ns) {StartInitialHandshake:157(28145423ns) {DropHandshakes:469(1988ns) {DropIncomingHandshake:441(539ns) DropOutgoingHandshake:456(354ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:476(2774ns) {SwitchToState:178(1832ns)}}}}}
2025-11-11T17:20:07.830117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518846340172774:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518850635140134:2112] held: no
2025-11-11T17:20:07.849856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518846340172773:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.850097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140151:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.854561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140156:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.902593Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518850635140156:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.910370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518846340172772:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:07.945868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518846340172774:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(27776925ns) {HandleHandshakeStatus:395(27774150ns) {LogHandshakeFail:472(11002ns) {UpdateErrorStateLog:544(3776ns)} DropOutgoingHandshake:456(159ns) ProcessPendingSessionEvents:495(27714762ns) {DropSessionEvent:512(15073ns) DropSessionEvent:512(4486ns) DropSessionEvent:512(4152ns) DropSessionEvent:512(3226ns) DropSessionEvent:512(3524ns) DropSessionEvent:512(3301ns) DropSessionEvent:512(7687209ns) DropSessionEvent:512(10110ns) DropSessionEvent:512(24865ns) DropSessionEvent:512(8634ns)}}}
2025-11-11T17:20:07.945989Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518850635140134:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:07.961759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518846340172772:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22999880ns) {IncomingHandshake:261(22997326ns) {DropIncomingHandshake:441(346ns) PrepareNewSessionHandshake:476(1562ns) {SwitchToState:178(698ns)} GenerateSessionVirtualId:988(246ns) IssueIncomingHandshakeReply:193(15463561ns) {SwitchToState:178(2373ns)}}}
2025-11-11T17:20:07.963417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518846340172774:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:07.963483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:07.963542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518846340172774:2050] [node 3] ICP22 created new session: [5:7571518850635140162:2050]
2025-11-11T17:20:07.963611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518850635140162:2050] [node 3] ICS09 handshake done sender: [5:7571518850635140134:2112] self: [5:7571518850635140145:0] peer: [3:7571518851007000744:0] socket: 403 qp: -1
2025-11-11T17:20:07.963657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518850635140162:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:07.963703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:07.963941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518846340172774:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(16148448ns) {HandleHandshakeStatus:329(16146372ns) {DropIncomingHandshake:441(2517ns) DropOutgoingHandshake:456(320ns) SwitchToState:178(763ns) ProcessPendingSessionEvents:495(279ns)}}
2025-11-11T17:20:07.973353Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:07.982011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518846340172772:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518850635140156:2121] held: no
2025-11-11T17:20:07.993993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140165:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:07.994225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518850635140164:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36152
2025-11-11T17:20:08.002907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518850635140153:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:08.002930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518850635140153:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:08.003857Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518850635140153:2120] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:08.004075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518850635140137:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:08.004164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:08.004219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518850635140137:2066] [node 6] ICP22 created new session: [5:7571518850635140167:2066]
2025-11-11T17:20:08.004266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518850635140167:2066] [node 6] ICS09 handshake done sender: [5:7571518850635140153:2120] self: [5:7571518850635140152:0] peer: [6:7571518852395611378:0] socket: 414 qp: -1
2025-11-11T17:20:08.004296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518850635140167:2066] [node 6] ICS10 traffic start
2025-11-11T17:20:08.004347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:08.005960Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518850635140156:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:08.006279Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518850635140144:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:08.030757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518846340172772:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.030851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:08.030900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518846340172772:2048] [node 1] ICP22 created new session: [5:7571518854930107465:2048]
2025-11-11T17:20:08.030942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518854930107465:2048] [node 1] ICS09 handshake done sender: [5:7571518850635140156:2121] self: [5:7571518850635140161:0] peer: [1:7571518851688518013:0] socket: 437 qp: -1
2025-11-11T17:20:08.030973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518854930107465:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:08.031038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:08.037771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518846340172773:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.047141Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:08.062367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:08.085718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518846340172773:2049] [node 2] ICP22 created new session: [5:7571518854930107467:2049]
2025-11-11T17:20:08.111223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518854930107467:2049] [node 2] ICS09 handshake done sender: [5:7571518850635140144:2116] self: [5:7571518850635140158:0] peer: [2:7571518849424074878:0] socket: 419 qp: -1
2025-11-11T17:20:08.111266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518854930107467:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:08.121719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:07.921794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518851688518027:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:07.937987Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518017:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:07.938013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518851688518017:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:07.950395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:07.954220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518851688518030:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16714
2025-11-11T17:20:07.969927Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518851688518017:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:08.134335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518853719042202:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:07.970680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518851688517985:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:07.970771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:08.134411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:07.970819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518851688517985:2093] [node 5] ICP22 created new session: [1:7571518851688518031:2093]
2025-11-11T17:20:08.136955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518845129107512:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(89519140ns) {HandleHandshakeStatus:329(89514300ns) {DropIncomingHandshake:441(2052ns) DropOutgoingHandshake:456(6241ns) SwitchToState:178(133ns) ProcessPendingSessionEvents:495(2187246ns) {ForwardSessionEventToSession:612(2140519ns) ForwardSessionEventToSession:612(4421ns) ForwardSessionEventToSession:612(2943ns) ForwardSessionEventToSession:612(9156ns) ForwardSessionEventToSession:612(2500ns) ForwardSessionEventToSession:612(2423ns) ForwardSessionEventToSession:612(5007ns) ForwardSessionEventToSession:612(2175ns) ForwardSessionEventToSession:612(5264ns) ForwardSessionEventToSession:612(2946ns)}}}
2025-11-11T17:20:07.970858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518851688518031:2093] [node 5] ICS09 handshake done sender: [1:7571518851688518017:2116] self: [1:7571518851688518013:0] peer: [5:7571518850635140161:0] socket: 440 qp: -1
2025-11-11T17:20:07.970895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518851688518031:2093] [node 5] ICS10 traffic start
2025-11-11T17:20:07.970924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:08.006562Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518851688518008:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:08.049997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518847393550642:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:08.050095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:08.050147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518847393550642:2048] [node 2] ICP22 created new session: [1:7571518855983485331:2048]
2025-11-11T17:20:08.050195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518855983485331:2048] [node 2] ICS09 handshake done sender: [1:7571518851688518008:2111] self: [1:7571518851688518007:0] peer: [2:7571518849424074891:0] socket: 422 qp: -1
2025-11-11T17:20:08.050229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518855983485331:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:08.050280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:08.058267Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518030:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:08.062229Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518015:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:08.062263Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518851688518015:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:08.086009Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518851688518015:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:08.087378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518851688518006:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:08.087440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:08.087475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518851688518006:2098] [node 6] ICP22 created new session: [1:7571518855983485335:2098]
2025-11-11T17:20:08.087513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518855983485335:2098] [node 6] ICS09 handshake done sender: [1:7571518851688518015:2115] self: [1:7571518851688518014:0] peer: [6:7571518856690578676:0] socket: 427 qp: -1
2025-11-11T17:20:08.087541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518855983485335:2098] [node 6] ICS10 traffic start
2025-11-11T17:20:08.087577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:08.093684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518847393550643:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(219766241ns) {HandleHandshakeStatus:329(219764183ns) {DropIncomingHandshake:441(18602968ns) DropOutgoingHandshake:456(1597ns) SwitchToState:178(480ns) ProcessPendingSessionEvents:495(5049414ns) {ForwardSessionEventToSession:612(5000265ns) ForwardSessionEventToSession:612(9622ns) ForwardSessionEventToSession:612(2947ns) ForwardSessionEventToSession:612(2556ns) ForwardSessionEventToSession:612(7329ns) ForwardSessionEventToSession:612(2769ns) ForwardSessionEventToSession:612(2540ns) ForwardSessionEventToSession:612(5199ns) ForwardSessionEventToSession:612(2377ns) ForwardSessionEventToSession:612(4294ns)}}}
2025-11-11T17:20:08.138416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518850817188158:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:08.132504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518856690578679:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50438
2025-11-11T17:20:08.134712Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518852395611370:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:08.058711Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518851007000776:2120] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:08.157707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518850817188158:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:08.145699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518853719042205:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:08.145794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:08.146320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518845129107513:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(86794486ns) {HandleHandshakeStatus:329(86791611ns) {DropIncomingHandshake:441(15046ns) DropOutgoingHandshake:456(660ns) SwitchToState:178(296ns) ProcessPendingSessionEvents:495(104848ns) {ForwardSessionEventToSession:612(49013ns) ForwardSessionEventToSession:612(7024ns) ForwardSessionEventToSession:612(3185ns) ForwardSessionEventToSession:612(3387ns) ForwardSessionEventToSession:612(11092ns) ForwardSessionEventToSession:612(2750ns) ForwardSessionEventToSession:612(5296ns) ForwardSessionEventToSession:612(5589ns) ForwardSessionEventToSession:612(2771ns) ForwardSessionEventToSession:612(4691ns)}}}
2025-11-11T17:20:08.149916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518848100643994:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.150003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:08.150053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518848100643994:2048] [node 1] ICP22 created new session: [6:7571518856690578685:2048]
2025-11-11T17:20:08.150098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518856690578685:2048] [node 1] ICS09 handshake done sender: [6:7571518852395611370:2116] self: [6:7571518856690578676:0] peer: [1:7571518851688518014:0] socket: 433 qp: -1
2025-11-11T17:20:08.150134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518856690578685:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:08.150169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:08.150592Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518852395611377:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:08.079894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518851007000755:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:08.079976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:08.080025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518851007000755:2075] [node 6] ICP22 created new session: [3:7571518855301968078:2075]
2025-11-11T17:20:08.080075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518855301968078:2075] [node 6] ICS09 handshake done sender: [3:7571518851007000776:2120] self: [3:7571518851007000775:0] peer: [6:7571518856690578675:0] socket: 446 qp: -1
2025-11-11T17:20:08.080108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518855301968078:2075] [node 6] ICS10 traffic start
2025-11-11T17:20:08.080139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:08.097727Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:08.101678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518846712033386:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518851007000774:2119] held: no
2025-11-11T17:20:08.132877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518846712033386:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(25521917ns) {HandleHandshakeStatus:395(25517827ns) {LogHandshakeFail:472(3803871ns) {UpdateErrorStateLog:544(9624ns)} DropOutgoingHandshake:456(753ns) ProcessPendingSessionEvents:495(21648709ns) {DropSessionEvent:512(21141ns) DropSessionEvent:512(9493ns) DropSessionEvent:512(4764ns) DropSessionEvent:512(4480ns) DropSessionEvent:512(8746284ns) DropSessionEvent:512(21094ns) DropSessionEvent:512(3929ns) DropSessionEvent:512(4095ns) DropSessionEvent:512(3821ns) DropSessionEvent:512(4740063ns)}}}
2025-11-11T17:20:08.153671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518855301968076:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23395
2025-11-11T17:20:08.162215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518846522220762:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:08.150709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:08.181819Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518851007000774:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:08.182128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518846712033386:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.182178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:08.182223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518846712033386:2049] [node 2] ICP22 created new session: [3:7571518855301968080:2049]
2025-11-11T17:20:08.182264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518855301968080:2049] [node 2] ICS09 handshake done sender: [3:7571518851007000774:2119] self: [3:7571518851007000779:0] peer: [2:7571518849424074882:0] socket: 412 qp: -1
2025-11-11T17:20:08.182299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518855301968080:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:08.182336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:08.162610Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518851688518020:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:08.185977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518855112155456:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-11T17:20:08.197831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518846340172773:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(124607042ns) {HandleHandshakeStatus:329(124604836ns) {DropIncomingHandshake:441(1743ns) DropOutgoingHandshake:456(8813ns) SwitchToState:178(743ns) ProcessPendingSessionEvents:495(165222ns) {ForwardSessionEventToSession:612(116130ns) ForwardSessionEventToSession:612(3407ns) ForwardSessionEventToSession:612(9313ns) ForwardSessionEventToSession:612(2816ns) ForwardSessionEventToSession:612(3108ns) ForwardSessionEventToSession:612(4350ns) ForwardSessionEventToSession:612(3724ns) ForwardSessionEventToSession:612(9045ns) ForwardSessionEventToSession:612(2760ns) ForwardSessionEventToSession:612(2478ns)}}}
2025-11-11T17:20:08.197934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518846340172773:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:08.154911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518848100643996:2050] [node 3] ICP22 created new session: [6:7571518856690578684:2050]
2025-11-11T17:20:08.162650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518851688518020:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:08.214565Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518850817188158:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:08.218028Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:08.174510Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518851688518020:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:08.213938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518851688518004:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:08.214027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:08.214078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518851688518004:2094] [node 4] ICP22 created new session: [1:7571518855983485338:2094]
2025-11-11T17:20:08.214760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518855983485338:2094] [node 4] ICS09 handshake done sender: [1:7571518851688518020:2118] self: [1:7571518851688518019:0] peer: [4:7571518855112155455:0] socket: 450 qp: -1
2025-11-11T17:20:08.231020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518855983485338:2094] [node 4] ICS10 traffic start
2025-11-11T17:20:08.231070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:08.231575Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518851688518004:2094] [node 4] ICP35 event processing took too much time PendingConnection:278(17372166ns) {HandleHandshakeStatus:329(17369895ns) {DropIncomingHandshake:441(21368ns) DropOutgoingHandshake:456(407ns) SwitchToState:178(1045ns) ProcessPendingSessionEvents:495(92778ns) {ForwardSessionEventToSession:612(39881ns) ForwardSessionEventToSession:612(2848ns) ForwardSessionEventToSession:612(10413ns) ForwardSessionEventToSession:612(3654ns) ForwardSessionEventToSession:612(8236ns) ForwardSessionEventToSession:612(5817ns) ForwardSessionEventToSession:612(3124ns) ForwardSessionEventToSession:612(5356ns) ForwardSessionEventToSession:612(2868ns) ForwardSessionEventToSession:612(3153ns)}}}
2025-11-11T17:20:08.161797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518856690578684:2050] [node 3] ICS09 handshake done sender: [6:7571518852395611375:2119] self: [6:7571518856690578675:0] peer: [3:7571518851007000775:0] socket: 448 qp: -1
2025-11-11T17:20:08.161845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518856690578684:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:08.161908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:08.162197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518848100643996:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(43357396ns) {HandleHandshakeStatus:329(43353799ns) {DropIncomingHandshake:441(687ns) DropOutgoingHandshake:456(370ns) SwitchToState:178(1110ns) ProcessPendingSessionEvents:495(934ns)}}
2025-11-11T17:20:08.162485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518848100643995:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.169854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:08.202179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518848100643995:2049] [node 2] ICP22 created new session: [6:7571518856690578688:2049]
2025-11-11T17:20:08.202237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518856690578688:2049] [node 2] ICS09 handshake done sender: [6:7571518852395611377:2120] self: [6:7571518856690578677:0] peer: [2:7571518849424074899:0] socket: 452 qp: -1
2025-11-11T17:20:08.202278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518856690578688:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:08.202326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:08.202547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518848100643995:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(39980868ns) {HandleHandshakeStatus:329(39978352ns) {DropIncomingHandshake:441(601ns) DropOutgoingHandshake:456(306ns) SwitchToState:178(332ns) ProcessPendingSessionEvents:495(842ns)}}
2025-11-11T17:20:08.229911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518846522220762:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:08.230031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:08.230088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518846522220762:2048] [node 1] ICP22 created new session: [4:7571518855112155457:2048]
2025-11-11T17:20:08.230167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518855112155457:2048] [node 1] ICS09 handshake done sender: [4:7571518850817188158:2123] self: [4:7571518855112155455:0] peer: [1:7571518851688518019:0] socket: 451 qp: -1
2025-11-11T17:20:08.230218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518855112155457:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:08.230267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:08.230605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518846522220762:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:08.294498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:08.294737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518853719042202:2048] [node 1] ICS01 socket: 424 reason# EndOfStream
2025-11-11T17:20:08.294790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518845129107512:2048] [node 1] ICP30 unregister session Session# [2:7571518853719042202:2048] VirtualId# [2:7571518849424074891:0]
2025-11-11T17:20:08.294834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518853719042202:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.294939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:08.294982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:08.305765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:08.305990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518851007000772:2048] [node 1] ICS01 socket: 399 reason# EndOfStream
2025-11-11T17:20:08.306027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518846712033385:2048] [node 1] ICP30 unregister session Session# [3:7571518851007000772:2048] VirtualId# [3:7571518851007000765:0]
2025-11-11T17:20:08.306064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518851007000772:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.306176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:08.306217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:08.313753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:08.313979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518854930107465:2048] [node 1] ICS01 socket: 437 reason# EndOfStream
2025-11-11T17:20:08.314028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518846340172772:2048] [node 1] ICP30 unregister session Session# [5:7571518854930107465:2048] VirtualId# [5:7571518850635140161:0]
2025-11-11T17:20:08.314063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518854930107465:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.314143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:08.314186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:08.313960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:08.314198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518855112155457:2048] [node 1] ICS01 socket: 451 reason# EndOfStream
2025-11-11T17:20:08.314229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518846522220762:2048] [node 1] ICP30 unregister session Session# [4:7571518855112155457:2048] VirtualId# [4:7571518855112155455:0]
2025-11-11T17:20:08.314256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518855112155457:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.314330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:08.314365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:08.317781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:08.318143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518856690578685:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-11-11T17:20:08.318188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518848100643994:2048] [node 1] ICP30 unregister session Session# [6:7571518856690578685:2048] VirtualId# [6:7571518856690578676:0]
2025-11-11T17:20:08.318222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518856690578685:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.318320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:08.318359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:08.441449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:08.441706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518855301968080:2049] [node 2] ICS01 socket: 412 reason# EndOfStream
2025-11-11T17:20:08.441748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518846712033386:2049] [node 2] ICP30 unregister session Session# [3:7571518855301968080:2049] VirtualId# [3:7571518851007000779:0]
2025-11-11T17:20:08.441787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518855301968080:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.441890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:08.441930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:08.465772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:08.465990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518854930107467:2049] [node 2] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-11T17:20:08.466022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518854930107467:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.466093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:08.466131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518854930107467:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:08.466649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:08.466833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518856690578688:2049] [node 2] ICS01 socket: 452 reason# EndOfStream
2025-11-11T17:20:08.466867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518848100643995:2049] [node 2] ICP30 unregister session Session# [6:7571518856690578688:2049] VirtualId# [6:7571518856690578677:0]
2025-11-11T17:20:08.466898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518856690578688:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.466956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:08.466995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:08.467834Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Couldn't connect to any resolved address
2025-11-11T17:20:08.468072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518846340172773:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:08.468113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518854930107467:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:08.468149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518846340172773:2049] [node 2] ICP30 unregister session Session# [5:7571518854930107467:2049] VirtualId# [5:7571518850635140158:0]
2025-11-11T17:20:08.468188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:08.468215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518846340172773:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:08.468231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:08.470030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:08.470215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518850817188156:2049] [node 2] ICS01 socket: 410 reason# EndOfStream
2025-11-11T17:20:08.470253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518846522220763:2049] [node 2] ICP30 unregister session Session# [4:7571518850817188156:2049] VirtualId# [4:7571518850817188153:0]
2025-11-11T17:20:08.470288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518850817188156:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.470381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:08.470428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:08.478247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518846340172773:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:08.569830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:08.570067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518850635140162:2050] [node 3] ICS01 socket: 403 reason# EndOfStream
2025-11-11T17:20:08.570113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518846340172774:2050] [node 3] ICP30 unregister session Session# [5:7571518850635140162:2050] VirtualId# [5:7571518850635140145:0]
2025-11-11T17:20:08.570154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518850635140162:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.570240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:08.570283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:08.570489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:08.570679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518850817188149:2050] [node 3] ICS01 socket: 391 reason# EndOfStream
2025-11-11T17:20:08.570723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518846522220764:2050] [node 3] ICP30 unregister session Session# [4:7571518850817188149:2050] VirtualId# [4:7571518850817188142:0]
2025-11-11T17:20:08.570754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518850817188149:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.570833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:08.570889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:08.573704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:08.573997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518856690578684:2050] [node 3] ICS01 socket: 448 reason# EndOfStream
2025-11-11T17:20:08.574054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518848100643996:2050] [node 3] ICP30 unregister session Session# [6:7571518856690578684:2050] VirtualId# [6:7571518856690578675:0]
2025-11-11T17:20:08.574105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518856690578684:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.574223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:08.574279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:08.653755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:08.653965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518850635140154:2067] [node 4] ICS01 socket: 404 reason# EndOfStream
2025-11-11T17:20:08.654004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518850635140138:2067] [node 4] ICP30 unregister session Session# [5:7571518850635140154:2067] VirtualId# [5:7571518850635140146:0]
2025-11-11T17:20:08.654037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518850635140154:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.654127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:08.654169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:08.653786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:08.654013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518852395611373:2065] [node 4] ICS07 socket disconnect 409 reason# EndOfStream
2025-11-11T17:20:08.654038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518852395611373:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.654123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:08.654169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518852395611373:2065] [node 4] ICS15 start handshake
2025-11-11T17:20:08.655070Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address
2025-11-11T17:20:08.655300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518852395611361:2065] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:08.655340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518852395611373:2065] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:08.655375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518852395611361:2065] [node 4] ICP30 unregister session Session# [6:7571518852395611373:2065] VirtualId# [6:7571518852395611367:0]
2025-11-11T17:20:08.655415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:08.655446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518852395611361:2065] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:08.655459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:08.665697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518852395611361:2065] [node 4] ICP33 wake up from error state
2025-11-11T17:20:08.851835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:08.852070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518856690578680:2061] [node 5] ICS01 socket: 441 reason# EndOfStream
2025-11-11T17:20:08.852128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518852395611306:2061] [node 5] ICP30 unregister session Session# [6:7571518856690578680:2061] VirtualId# [6:7571518852395611378:0]
2025-11-11T17:20:08.852163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518856690578680:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:08.852268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:08.852311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:09.069892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518860928936482:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:09.069951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518860928936483:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:09.116385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518859601390887:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:09.116436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518859601390888:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:09.165885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518857735595635:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:09.165944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518857735595636:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:09.201946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518859123064533:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:09.201993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518859123064534:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:09.202023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518859123064535:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:09.254399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518860306561312:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:09.254449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518860306561313:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:09.254475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518860306561314:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:09.572063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518861495348672:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:09.572130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518861495348673:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:09.572162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518861495348674:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:09.572832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518861495348689:2059] [node 4] ICP01 ready to work
2025-11-11T17:20:09.573131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518861495348690:2061] [node 5] ICP01 ready to work
2025-11-11T17:20:09.653882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518859601390946:2068] [node 5] ICP01 ready to work
2025-11-11T17:20:09.654301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518859601390947:2070] [node 6] ICP01 ready to work
2025-11-11T17:20:09.654609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518859601390948:2073] [node 4] ICP01 ready to work
2025-11-11T17:20:09.660762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518859123064596:2066] [node 5] ICP01 ready to work
2025-11-11T17:20:09.703524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518859601390946:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(48575127ns) {Configure:104(48569892ns) {ProcessConfigured:135(91737ns) {StartInitialHandshake:157(88878ns) {DropHandshakes:469(2594ns) {DropIncomingHandshake:441(509ns) DropOutgoingHandshake:456(490ns)} GenerateSessionVirtualId:988(912ns) PrepareNewSessionHandshake:476(2458ns) {SwitchToState:178(971ns)}}}}}
2025-11-11T17:20:09.709653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518860306561354:2063] [node 6] ICP01 ready to work
2025-11-11T17:20:09.739008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518857735595699:2065] [node 4] ICP01 ready to work
2025-11-11T17:20:09.741703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518859123064597:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:09.708484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518860928936546:2080] [node 5] ICP01 ready to work
2025-11-11T17:20:09.790778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518859601390947:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(85502284ns) {Configure:104(85497618ns) {ProcessConfigured:135(68383ns) {StartInitialHandshake:157(65899ns) {DropHandshakes:469(3053ns) {DropIncomingHandshake:441(614ns) DropOutgoingHandshake:456(687ns)} GenerateSessionVirtualId:988(1359ns) PrepareNewSessionHandshake:476(1839ns) {SwitchToState:178(543ns)}}}}}
2025-11-11T17:20:09.814015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518857735595700:2066] [node 6] ICP01 ready to work
2025-11-11T17:20:09.813725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518860306561376:2064] [node 4] ICP01 ready to work
2025-11-11T17:20:09.814178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518860928936547:2081] [node 4] ICP01 ready to work
2025-11-11T17:20:09.814613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518860928936548:2082] [node 6] ICP01 ready to work
2025-11-11T17:20:09.847187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518861495348689:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(273711139ns) {Configure:104(273706266ns) {ProcessConfigured:135(13163240ns) {StartInitialHandshake:157(13159831ns) {DropHandshakes:469(1951ns) {DropIncomingHandshake:441(405ns) DropOutgoingHandshake:456(428ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:476(4827ns) {SwitchToState:178(2029ns)}}}}}
2025-11-11T17:20:09.882131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518857735595701:2070] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.891848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518859601390948:2073] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(49547079ns) {Configure:104(49543570ns) {ProcessConfigured:135(47650299ns) {StartInitialHandshake:157(47648462ns) {DropHandshakes:469(1400ns) {DropIncomingHandshake:441(192ns) DropOutgoingHandshake:456(344ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:476(5855ns) {SwitchToState:178(3360ns)}}}}}
2025-11-11T17:20:09.891515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518859123064611:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:09.878565Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518861495348690:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31090801ns) {Configure:104(31085647ns) {ProcessConfigured:135(61493ns) {StartInitialHandshake:157(58911ns) {DropHandshakes:469(1869ns) {DropIncomingHandshake:441(411ns) DropOutgoingHandshake:456(305ns)} GenerateSessionVirtualId:988(559ns) PrepareNewSessionHandshake:476(1593ns) {SwitchToState:178(571ns)}}}}}
2025-11-11T17:20:09.862216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518860928936547:2081] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45407990ns) {Configure:104(45404191ns) {ProcessConfigured:135(68999ns) {StartInitialHandshake:157(65731ns) {DropHandshakes:469(2590ns) {DropIncomingHandshake:441(611ns) DropOutgoingHandshake:456(365ns)} GenerateSessionVirtualId:988(967ns) PrepareNewSessionHandshake:476(1306ns) {SwitchToState:178(458ns)}}}}}
2025-11-11T17:20:09.910549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518857735595636:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24973209ns) {Configure:104(24970498ns) {ProcessConfigured:135(60671ns) {StartInitialHandshake:157(58604ns) {DropHandshakes:469(2535ns) {DropIncomingHandshake:441(418ns) DropOutgoingHandshake:456(737ns)} GenerateSessionVirtualId:988(718ns) PrepareNewSessionHandshake:476(1664ns) {SwitchToState:178(471ns)}}}}}
2025-11-11T17:20:09.927654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518860306561312:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53594050ns) {Configure:104(53587940ns) {ProcessConfigured:135(67443ns) {StartInitialHandshake:157(65673ns) {DropHandshakes:469(2306ns) {DropIncomingHandshake:441(625ns) DropOutgoingHandshake:456(373ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:476(1312ns) {SwitchToState:178(509ns)}}}}}
2025-11-11T17:20:09.904340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518861495348672:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23373819ns) {Configure:104(23369415ns) {ProcessConfigured:135(2483839ns) {StartInitialHandshake:157(2481210ns) {DropHandshakes:469(4720ns) {DropIncomingHandshake:441(450ns) DropOutgoingHandshake:456(345ns)} GenerateSessionVirtualId:988(1226ns) PrepareNewSessionHandshake:476(4161ns) {SwitchToState:178(1837ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.947273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518859123064612:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:09.947330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518859123064613:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:09.946034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518861495348673:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(41444649ns) {Configure:104(41439132ns) {ProcessConfigured:135(69008ns) {StartInitialHandshake:157(66802ns) {DropHandshakes:469(2429ns) {DropIncomingHandshake:441(396ns) DropOutgoingHandshake:456(448ns)} GenerateSessionVirtualId:988(640ns) PrepareNewSessionHandshake:476(1683ns) {SwitchToState:178(524ns)}}}}}
2025-11-11T17:20:09.946423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518861495348741:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:09.964294Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518860306561354:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36365141ns) {Configure:104(36361240ns) {ProcessConfigured:135(68054ns) {StartInitialHandshake:157(65265ns) {DropHandshakes:469(2179ns) {DropIncomingHandshake:441(592ns) DropOutgoingHandshake:456(330ns)} GenerateSessionVirtualId:988(699ns) PrepareNewSessionHandshake:476(1604ns) {SwitchToState:178(527ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.965835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518861495348741:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:09.966177Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518859123064613:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.978660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518857735595715:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.981833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518859123064599:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:09.981870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518859123064599:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:09.982924Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518859123064599:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:09.976264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518859601390963:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52278
2025-11-11T17:20:09.976535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518859601390964:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52278
2025-11-11T17:20:09.976966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518859601390965:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52278
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.983218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518859123064597:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:09.983270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:09.983310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518859123064597:2067] [node 6] ICP22 created new session: [4:7571518859123064616:2067]
2025-11-11T17:20:09.983358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518859123064616:2067] [node 6] ICS09 handshake done sender: [4:7571518859123064599:2110] self: [4:7571518859123064598:0] peer: [6:7571518861495348751:0] socket: 396 qp: -1
2025-11-11T17:20:09.983387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518859123064616:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:09.983416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.977717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518861495348689:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:09.997910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518860928936562:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42344
2025-11-11T17:20:09.994369Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518859601390963:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:09.985624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518860306561385:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.022651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518860928936563:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42344
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.023984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518860928936564:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42344
2025-11-11T17:20:10.024056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518860928936565:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42344
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:09.985951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518860306561386:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-11-11T17:20:09.986369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518860306561389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-11-11T17:20:10.014290Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518857735595715:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.044814Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936562:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.023530Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518859123064612:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:09.978186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518861495348752:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.078100Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518857735595703:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.078138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518857735595703:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.024702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518863418031914:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:10.081982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936563:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.082393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936565:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.082444Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518860306561385:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.082565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518860306561312:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.106300Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:10.104163Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936550:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.107414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518862030563013:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53061
2025-11-11T17:20:10.086265Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518859601390964:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.121764Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518857735595703:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:10.024779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518859123064535:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.138541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518861495348689:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518861495348741:2112] held: no
2025-11-11T17:20:10.138829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316049:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.139446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316050:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.139759Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518861495348741:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:10.140292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518861495348689:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.140360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:10.140407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518861495348689:2059] [node 4] ICP22 created new session: [6:7571518865790316052:2059]
2025-11-11T17:20:10.140452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518865790316052:2059] [node 4] ICS09 handshake done sender: [6:7571518861495348741:2112] self: [6:7571518861495348751:0] peer: [4:7571518859123064598:0] socket: 402 qp: -1
2025-11-11T17:20:10.140486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518865790316052:2059] [node 4] ICS10 traffic start
2025-11-11T17:20:10.140532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.086463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518859601390887:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.104378Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518863418031914:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.104203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518860928936550:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.103839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518860306561389:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.125717Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518859601390953:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.125761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518859601390953:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.126497Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518859601390965:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.105211Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936564:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.105292Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936554:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.105308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518860928936554:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.114303Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518860928936550:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:10.104015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518860306561386:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.106635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518859123064611:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.141986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518859123064533:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.142610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518863418031917:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:10.143379Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518859123064612:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:10.151375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518859123064535:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.151437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:10.151484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518859123064535:2050] [node 3] ICP22 created new session: [4:7571518863418031920:2050]
2025-11-11T17:20:10.151530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518863418031920:2050] [node 3] ICS09 handshake done sender: [4:7571518859123064612:2116] self: [4:7571518863418031916:0] peer: [3:7571518857735595702:0] socket: 413 qp: -1
2025-11-11T17:20:10.151573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518863418031920:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:10.151615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:10.160201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518857735595699:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.160289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:10.160339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518857735595699:2065] [node 4] ICP22 created new session: [3:7571518862030563015:2065]
2025-11-11T17:20:10.160380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518862030563015:2065] [node 4] ICS09 handshake done sender: [3:7571518857735595703:2111] self: [3:7571518857735595702:0] peer: [4:7571518863418031916:0] socket: 407 qp: -1
2025-11-11T17:20:10.160416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518862030563015:2065] [node 4] ICS10 traffic start
2025-11-11T17:20:10.160454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:10.161106Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518862030563013:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.161571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518857735595699:2065] [node 4] ICP18 handshake (actor [3:7571518862030563013:2117]) is too old
2025-11-11T17:20:10.158255Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518860928936554:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.104585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518860306561313:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.114020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518864601528692:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-11-11T17:20:10.175956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518861495348752:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.176126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518861495348673:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.176319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518863896358267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52278
2025-11-11T17:20:10.178380Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518859601390953:2109] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:10.178735Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936552:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.178770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518860928936552:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.179197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518860928936546:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.179289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:10.179330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518860928936546:2080] [node 5] ICP22 created new session: [1:7571518865223903869:2080]
2025-11-11T17:20:10.179381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518865223903869:2080] [node 5] ICS09 handshake done sender: [1:7571518860928936550:2111] self: [1:7571518860928936549:0] peer: [5:7571518864601528689:0] socket: 399 qp: -1
2025-11-11T17:20:10.179410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518865223903869:2080] [node 5] ICS10 traffic start
2025-11-11T17:20:10.179441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:10.179826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518860928936482:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.179896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:10.179932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518860928936482:2048] [node 2] ICP22 created new session: [1:7571518865223903871:2048]
2025-11-11T17:20:10.179972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518865223903871:2048] [node 2] ICS09 handshake done sender: [1:7571518860928936554:2113] self: [1:7571518860928936553:0] peer: [2:7571518863896358266:0] socket: 401 qp: -1
2025-11-11T17:20:10.180004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518865223903871:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:10.180049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:10.195196Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518859601390958:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.195234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518859601390958:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.195826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518859601390946:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.195896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:10.195945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518859601390946:2068] [node 5] ICP22 created new session: [2:7571518863896358270:2068]
2025-11-11T17:20:10.195992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518863896358270:2068] [node 5] ICS09 handshake done sender: [2:7571518859601390953:2109] self: [2:7571518859601390952:0] peer: [5:7571518864601528690:0] socket: 420 qp: -1
2025-11-11T17:20:10.196025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518863896358270:2068] [node 5] ICS10 traffic start
2025-11-11T17:20:10.196059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:10.196527Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518859601390964:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:10.197062Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26789) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.198839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518865223903868:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42344
2025-11-11T17:20:10.199962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518860306561385:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:10.201781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518865790316050:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.201814Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518860928936552:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.202171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518860306561312:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26789) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518860306561385:2114] held: no
2025-11-11T17:20:10.202277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518860306561312:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.202353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:10.202396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518860306561312:2048] [node 1] ICP22 created new session: [5:7571518864601528696:2048]
2025-11-11T17:20:10.202465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518864601528696:2048] [node 1] ICS09 handshake done sender: [5:7571518860306561385:2114] self: [5:7571518864601528689:0] peer: [1:7571518860928936549:0] socket: 418 qp: -1
2025-11-11T17:20:10.202503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518864601528696:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:10.202557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:10.204410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518863418031923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:10.204817Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518859123064611:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:10.201979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518862030563018:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53061
2025-11-11T17:20:10.213836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518859123064533:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.213905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:10.213961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518859123064533:2048] [node 1] ICP22 created new session: [4:7571518863418031924:2048]
2025-11-11T17:20:10.214007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518863418031924:2048] [node 1] ICS09 handshake done sender: [4:7571518859123064611:2115] self: [4:7571518863418031918:0] peer: [1:7571518860928936551:0] socket: 404 qp: -1
2025-11-11T17:20:10.214045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518863418031924:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:10.214085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.216126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518861495348690:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.216220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316055:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.214844Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26789) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:10.216793Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518865223903868:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.216847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518860928936547:2081] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.216915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:10.216274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316056:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.216959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518860928936547:2081] [node 4] ICP22 created new session: [1:7571518865223903876:2081]
2025-11-11T17:20:10.217002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518865223903876:2081] [node 4] ICS09 handshake done sender: [1:7571518860928936552:2112] self: [1:7571518860928936551:0] peer: [4:7571518863418031918:0] socket: 400 qp: -1
2025-11-11T17:20:10.217033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518865223903876:2081] [node 4] ICS10 traffic start
2025-11-11T17:20:10.217079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.219829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518864601528693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-11-11T17:20:10.218877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518859601390887:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.218942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:10.218983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518859601390887:2048] [node 1] ICP22 created new session: [2:7571518863896358273:2048]
2025-11-11T17:20:10.219029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518863896358273:2048] [node 1] ICS09 handshake done sender: [2:7571518859601390964:2116] self: [2:7571518863896358266:0] peer: [1:7571518860928936553:0] socket: 406 qp: -1
2025-11-11T17:20:10.219057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518863896358273:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:10.219082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:10.219250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518859601390887:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.222215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518862030563019:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53061
2025-11-11T17:20:10.222458Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518859601390958:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.224761Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518862030563018:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.224909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518857735595635:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.224470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518863418031926:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:10.223041Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518861495348752:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:10.225467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518859601390947:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.225515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:10.229907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936556:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.229947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518860928936556:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.231155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518861495348673:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.241691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518862030563021:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53061
2025-11-11T17:20:10.225559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518859601390947:2070] [node 6] ICP22 created new session: [2:7571518863896358277:2070]
2025-11-11T17:20:10.233853Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518860928936556:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.245374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518864601528694:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.245549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518864601528695:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-11-11T17:20:10.253957Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518860306561380:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.254005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518860306561380:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.257894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518860928936483:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.258023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:10.225622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518863896358277:2070] [node 6] ICS09 handshake done sender: [2:7571518859601390958:2112] self: [2:7571518859601390957:0] peer: [6:7571518865790316054:0] socket: 422 qp: -1
2025-11-11T17:20:10.242012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:10.258082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518860928936483:2049] [node 3] ICP22 created new session: [1:7571518865223903880:2049]
2025-11-11T17:20:10.258127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518865223903880:2049] [node 3] ICS09 handshake done sender: [1:7571518860928936556:2114] self: [1:7571518860928936555:0] peer: [3:7571518862030563020:0] socket: 445 qp: -1
2025-11-11T17:20:10.258169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518865223903880:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:10.258222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:10.225651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518863896358277:2070] [node 6] ICS10 traffic start
2025-11-11T17:20:10.225838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:10.245651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518861495348673:2049] [node 2] ICP22 created new session: [6:7571518865790316058:2049]
2025-11-11T17:20:10.245715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518865790316058:2049] [node 2] ICS09 handshake done sender: [6:7571518861495348752:2116] self: [6:7571518865790316054:0] peer: [2:7571518859601390957:0] socket: 423 qp: -1
2025-11-11T17:20:10.245768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518865790316058:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:10.245838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:10.250199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518861495348673:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(23970910ns) {HandleHandshakeStatus:329(23969139ns) {DropIncomingHandshake:441(1005ns) DropOutgoingHandshake:456(7780ns) SwitchToState:178(392ns) ProcessPendingSessionEvents:495(3922811ns) {ForwardSessionEventToSession:612(3871454ns) ForwardSessionEventToSession:612(9448ns) ForwardSessionEventToSession:612(3719ns) ForwardSessionEventToSession:612(5147ns) ForwardSessionEventToSession:612(3026ns) ForwardSessionEventToSession:612(6460ns) ForwardSessionEventToSession:612(4912ns) ForwardSessionEventToSession:612(2841ns) ForwardSessionEventToSession:612(2309ns) ForwardSessionEventToSession:612(4940ns)}}}
2025-11-11T17:20:10.258985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518865790316055:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.259404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518861495348672:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.274168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518859601390947:2070] [node 6] ICP35 event processing took too much time PendingConnection:278(48483964ns) {HandleHandshakeStatus:329(48481975ns) {DropIncomingHandshake:441(186ns) DropOutgoingHandshake:456(523ns) SwitchToState:178(308ns) ProcessPendingSessionEvents:495(18964221ns) {ForwardSessionEventToSession:612(10616031ns) ForwardSessionEventToSession:612(8267166ns) ForwardSessionEventToSession:612(5693ns) ForwardSessionEventToSession:612(2752ns) ForwardSessionEventToSession:612(2869ns) ForwardSessionEventToSession:612(4612ns) ForwardSessionEventToSession:612(2173ns) ForwardSessionEventToSession:612(3007ns) ForwardSessionEventToSession:612(4787ns) ForwardSessionEventToSession:612(5697ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.270078Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518860928936558:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.270115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518860928936558:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.274862Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518860928936558:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.282748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316061:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.302872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316062:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.305821Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518865790316055:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:10.306329Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518865790316050:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:10.277560Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518864601528695:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.307439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518861495348672:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.307519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:10.307579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518861495348672:2048] [node 1] ICP22 created new session: [6:7571518865790316064:2048]
2025-11-11T17:20:10.307622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518865790316064:2048] [node 1] ICS09 handshake done sender: [6:7571518865790316055:2119] self: [6:7571518865790316060:0] peer: [1:7571518860928936557:0] socket: 448 qp: -1
2025-11-11T17:20:10.307660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518865790316064:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:10.307710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:10.308099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518861495348690:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.308135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:10.308206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518861495348690:2061] [node 5] ICP22 created new session: [6:7571518865790316066:2061]
2025-11-11T17:20:10.308241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518865790316066:2061] [node 5] ICS09 handshake done sender: [6:7571518865790316050:2118] self: [6:7571518865790316057:0] peer: [5:7571518860306561379:0] socket: 433 qp: -1
2025-11-11T17:20:10.308270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518865790316066:2061] [node 5] ICS10 traffic start
2025-11-11T17:20:10.308303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:10.285756Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26789) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.324534Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518863418031926:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.326305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518859123064534:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.324873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316063:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.277677Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518860306561389:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:10.286679Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518862030563018:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:10.286953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518857735595635:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26789) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518862030563018:2118] held: no
2025-11-11T17:20:10.287048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518857735595635:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.293683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518864601528694:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.287117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:10.287160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518857735595635:2048] [node 1] ICP22 created new session: [3:7571518862030563023:2048]
2025-11-11T17:20:10.287204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518862030563023:2048] [node 1] ICS09 handshake done sender: [3:7571518862030563018:2118] self: [3:7571518862030563020:0] peer: [1:7571518860928936555:0] socket: 426 qp: -1
2025-11-11T17:20:10.295116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518862030563023:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:10.295154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:10.302409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518862030563022:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53061
2025-11-11T17:20:10.329680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518862030563019:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.337909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518857735595636:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.345241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518859601390962:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.333754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518860928936548:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.345306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518859601390962:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.293791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518860306561313:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.347603Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518865790316063:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.348023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518861495348674:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.357209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:10.357280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518860928936548:2082] [node 6] ICP22 created new session: [1:7571518865223903883:2082]
2025-11-11T17:20:10.366014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518859601390960:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.293905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:10.354066Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518859123064607:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.354112Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518859123064607:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.366051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518859601390960:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.362017Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518857735595711:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.361981Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518859123064607:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:10.362058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518857735595711:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.365513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518865223903883:2082] [node 6] ICS09 handshake done sender: [1:7571518860928936558:2115] self: [1:7571518860928936557:0] peer: [6:7571518865790316060:0] socket: 446 qp: -1
2025-11-11T17:20:10.293960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518860306561313:2049] [node 2] ICP22 created new session: [5:7571518864601528700:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.385732Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518859601390962:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:10.386499Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518862030563022:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.386606Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518857735595709:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:10.386625Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518857735595709:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:10.387984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518862030563027:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53061
2025-11-11T17:20:10.393701Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518857735595711:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:10.394164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518865790316069:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48694
2025-11-11T17:20:10.385429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518865223903883:2082] [node 6] ICS10 traffic start
2025-11-11T17:20:10.410040Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518859601390960:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:10.410429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518859601390948:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.410507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:10.410559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518859601390948:2073] [node 4] ICP22 created new session: [2:7571518863896358281:2073]
2025-11-11T17:20:10.410600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518863896358281:2073] [node 4] ICS09 handshake done sender: [2:7571518859601390962:2114] self: [2:7571518859601390961:0] peer: [4:7571518863418031927:0] socket: 451 qp: -1
2025-11-11T17:20:10.410635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518863896358281:2073] [node 4] ICS10 traffic start
2025-11-11T17:20:10.410687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:10.411086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518859601390888:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.411160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:10.411196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518859601390888:2049] [node 3] ICP22 created new session: [2:7571518863896358283:2049]
2025-11-11T17:20:10.411238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518863896358283:2049] [node 3] ICS09 handshake done sender: [2:7571518859601390960:2113] self: [2:7571518859601390959:0] peer: [3:7571518862030563026:0] socket: 421 qp: -1
2025-11-11T17:20:10.411281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518863896358283:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:10.411329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:10.293999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518864601528700:2049] [node 2] ICS09 handshake done sender: [5:7571518860306561389:2117] self: [5:7571518864601528690:0] peer: [2:7571518859601390952:0] socket: 424 qp: -1
2025-11-11T17:20:10.415120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518859123064596:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.415184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:10.415224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518859123064596:2066] [node 5] ICP22 created new session: [4:7571518863418031930:2066]
2025-11-11T17:20:10.415264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518863418031930:2066] [node 5] ICS09 handshake done sender: [4:7571518859123064607:2114] self: [4:7571518859123064606:0] peer: [5:7571518864601528704:0] socket: 432 qp: -1
2025-11-11T17:20:10.415291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518863418031930:2066] [node 5] ICS10 traffic start
2025-11-11T17:20:10.415318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:10.415861Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:10.385803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:10.294040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518864601528700:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:10.394930Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518865790316063:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:10.395100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518861495348674:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.395157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:10.402401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518861495348674:2050] [node 3] ICP22 created new session: [6:7571518865790316070:2050]
2025-11-11T17:20:10.402464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518865790316070:2050] [node 3] ICS09 handshake done sender: [6:7571518865790316063:2123] self: [6:7571518865790316068:0] peer: [3:7571518857735595710:0] socket: 459 qp: -1
2025-11-11T17:20:10.402503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518865790316070:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:10.402551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:10.294078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:10.294846Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518860306561380:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:10.307187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518860306561354:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.307275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:10.307314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518860306561354:2063] [node 6] ICP22 created new session: [5:7571518864601528702:2063]
2025-11-11T17:20:10.322523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518864601528702:2063] [node 6] ICS09 handshake done sender: [5:7571518860306561380:2111] self: [5:7571518860306561379:0] peer: [6:7571518865790316057:0] socket: 427 qp: -1
2025-11-11T17:20:10.322578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518864601528702:2063] [node 6] ICS10 traffic start
2025-11-11T17:20:10.322637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:10.322948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518860306561376:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.346334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518860306561314:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:10.370101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518864601528706:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-11-11T17:20:10.377984Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518864601528695:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:10.378493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518860306561376:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.378555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:10.378600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518860306561376:2064] [node 4] ICP22 created new session: [5:7571518864601528707:2064]
2025-11-11T17:20:10.378648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518864601528707:2064] [node 4] ICS09 handshake done sender: [5:7571518864601528695:2121] self: [5:7571518864601528704:0] peer: [4:7571518859123064606:0] socket: 444 qp: -1
2025-11-11T17:20:10.378685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518864601528707:2064] [node 4] ICS10 traffic start
2025-11-11T17:20:10.378718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:10.395654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518864601528709:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-11-11T17:20:10.418406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518860928936548:2082] [node 6] ICP35 event processing took too much time PendingConnection:278(91493794ns) {HandleHandshakeStatus:329(88076908ns) {DropIncomingHandshake:441(17302ns) DropOutgoingHandshake:456(4425ns) SwitchToState:178(680ns) ProcessPendingSessionEvents:495(12421080ns) {ForwardSessionEventToSession:612(11818453ns) ForwardSessionEventToSession:612(4633ns) ForwardSessionEventToSession:612(557890ns) ForwardSessionEventToSession:612(3425ns) ForwardSessionEventToSession:612(8183ns) ForwardSessionEventToSession:612(4986ns) ForwardSessionEventToSession:612(2711ns) ForwardSessionEventToSession:612(4787ns) ForwardSessionEventToSession:612(2683ns) ForwardSessionEventToSession:612(2418ns)}}}
2025-11-11T17:20:10.419026Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518857735595709:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:10.427397Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518864601528694:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:10.427803Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:10.429332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518860306561314:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.429393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:10.429441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518860306561314:2050] [node 3] ICP22 created new session: [5:7571518864601528710:2050]
2025-11-11T17:20:10.429480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518864601528710:2050] [node 3] ICS09 handshake done sender: [5:7571518864601528694:2120] self: [5:7571518864601528705:0] peer: [3:7571518857735595708:0] socket: 443 qp: -1
2025-11-11T17:20:10.429512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518864601528710:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:10.429542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:10.431537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518857735595700:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.431625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:10.431670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518857735595700:2066] [node 6] ICP22 created new session: [3:7571518862030563031:2066]
2025-11-11T17:20:10.431718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518862030563031:2066] [node 6] ICS09 handshake done sender: [3:7571518857735595711:2115] self: [3:7571518857735595710:0] peer: [6:7571518865790316068:0] socket: 408 qp: -1
2025-11-11T17:20:10.431755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518862030563031:2066] [node 6] ICS10 traffic start
2025-11-11T17:20:10.431782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:10.432275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518857735595701:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:10.432312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:10.432346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518857735595701:2070] [node 5] ICP22 created new session: [3:7571518862030563033:2070]
2025-11-11T17:20:10.432384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518862030563033:2070] [node 5] ICS09 handshake done sender: [3:7571518857735595709:2114] self: [3:7571518857735595708:0] peer: [5:7571518864601528705:0] socket: 442 qp: -1
2025-11-11T17:20:10.432407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518862030563033:2070] [node 5] ICS10 traffic start
2025-11-11T17:20:10.432427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:10.432836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518857735595636:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518862030563019:2119] held: no
2025-11-11T17:20:10.433765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518859123064534:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518863418031926:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:10.434091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518863418031929:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7456
2025-11-11T17:20:10.434528Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518863418031926:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:10.434745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518859123064534:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.434781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:10.434818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518859123064534:2049] [node 2] ICP22 created new session: [4:7571518863418031932:2049]
2025-11-11T17:20:10.434861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518863418031932:2049] [node 2] ICS09 handshake done sender: [4:7571518863418031926:2121] self: [4:7571518863418031927:0] peer: [2:7571518859601390961:0] socket: 397 qp: -1
2025-11-11T17:20:10.434889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518863418031932:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:10.434915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:10.436255Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518862030563019:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:10.436658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518857735595636:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:10.436695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:10.436731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518857735595636:2049] [node 2] ICP22 created new session: [3:7571518862030563035:2049]
2025-11-11T17:20:10.436769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518862030563035:2049] [node 2] ICS09 handshake done sender: [3:7571518862030563019:2119] self: [3:7571518862030563026:0] peer: [2:7571518859601390959:0] socket: 403 qp: -1
2025-11-11T17:20:10.436794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518862030563035:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:10.436817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:10.497158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:10.497394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518863896358273:2048] [node 1] ICS01 socket: 406 reason# EndOfStream
2025-11-11T17:20:10.497435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518859601390887:2048] [node 1] ICP30 unregister session Session# [2:7571518863896358273:2048] VirtualId# [2:7571518863896358266:0]
2025-11-11T17:20:10.497465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518863896358273:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.497531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:10.497571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:10.506935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:10.507182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518862030563023:2048] [node 1] ICS01 socket: 426 reason# EndOfStream
2025-11-11T17:20:10.507228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518857735595635:2048] [node 1] ICP30 unregister session Session# [3:7571518862030563023:2048] VirtualId# [3:7571518862030563020:0]
2025-11-11T17:20:10.507264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518862030563023:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.507333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:10.507371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:10.532305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:10.532531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518864601528696:2048] [node 1] ICS01 socket: 418 reason# EndOfStream
2025-11-11T17:20:10.532572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518860306561312:2048] [node 1] ICP30 unregister session Session# [5:7571518864601528696:2048] VirtualId# [5:7571518864601528689:0]
2025-11-11T17:20:10.532607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518864601528696:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.532668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:10.532706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:10.537741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:10.537966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518863418031924:2048] [node 1] ICS01 socket: 404 reason# EndOfStream
2025-11-11T17:20:10.538008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518859123064533:2048] [node 1] ICP30 unregister session Session# [4:7571518863418031924:2048] VirtualId# [4:7571518863418031918:0]
2025-11-11T17:20:10.538043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518863418031924:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.538120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:10.538159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:10.538278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:10.538513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518865790316064:2048] [node 1] ICS01 socket: 448 reason# EndOfStream
2025-11-11T17:20:10.538566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518861495348672:2048] [node 1] ICP30 unregister session Session# [6:7571518865790316064:2048] VirtualId# [6:7571518865790316060:0]
2025-11-11T17:20:10.538605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518865790316064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.538689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:10.538733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:10.633046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:10.633285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518862030563035:2049] [node 2] ICS01 socket: 403 reason# EndOfStream
2025-11-11T17:20:10.633330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518857735595636:2049] [node 2] ICP30 unregister session Session# [3:7571518862030563035:2049] VirtualId# [3:7571518862030563026:0]
2025-11-11T17:20:10.633361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518862030563035:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.633478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:10.633532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:10.641738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:10.641967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518864601528700:2049] [node 2] ICS01 socket: 424 reason# EndOfStream
2025-11-11T17:20:10.642011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518860306561313:2049] [node 2] ICP30 unregister session Session# [5:7571518864601528700:2049] VirtualId# [5:7571518864601528690:0]
2025-11-11T17:20:10.642047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518864601528700:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.642156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:10.642212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:10.648379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:10.648625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518863418031932:2049] [node 2] ICS01 socket: 397 reason# EndOfStream
2025-11-11T17:20:10.648667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518859123064534:2049] [node 2] ICP30 unregister session Session# [4:7571518863418031932:2049] VirtualId# [4:7571518863418031927:0]
2025-11-11T17:20:10.648701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518863418031932:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.648795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:10.648838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:10.649174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:10.649369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518865790316058:2049] [node 2] ICS01 socket: 423 reason# EndOfStream
2025-11-11T17:20:10.649401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518861495348673:2049] [node 2] ICP30 unregister session Session# [6:7571518865790316058:2049] VirtualId# [6:7571518865790316054:0]
2025-11-11T17:20:10.649425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518865790316058:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.649482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:10.649518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:10.719136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:10.719362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518863418031920:2050] [node 3] ICS01 socket: 413 reason# EndOfStream
2025-11-11T17:20:10.719400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518859123064535:2050] [node 3] ICP30 unregister session Session# [4:7571518863418031920:2050] VirtualId# [4:7571518863418031916:0]
2025-11-11T17:20:10.719433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518863418031920:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.719501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:10.719542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:10.725712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:10.725947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518865790316070:2050] [node 3] ICS01 socket: 459 reason# EndOfStream
2025-11-11T17:20:10.725979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518861495348674:2050] [node 3] ICP30 unregister session Session# [6:7571518865790316070:2050] VirtualId# [6:7571518865790316068:0]
2025-11-11T17:20:10.726008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518865790316070:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.726058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:10.726095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:10.725700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:10.725907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518864601528710:2050] [node 3] ICS01 socket: 443 reason# EndOfStream
2025-11-11T17:20:10.725942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518860306561314:2050] [node 3] ICP30 unregister session Session# [5:7571518864601528710:2050] VirtualId# [5:7571518864601528705:0]
2025-11-11T17:20:10.725981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518864601528710:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.726058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:10.726095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:10.829334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:10.829564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518864601528707:2064] [node 4] ICS01 socket: 444 reason# EndOfStream
2025-11-11T17:20:10.829613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518860306561376:2064] [node 4] ICP30 unregister session Session# [5:7571518864601528707:2064] VirtualId# [5:7571518864601528704:0]
2025-11-11T17:20:10.829649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518864601528707:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.829712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:10.829759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:10.832873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:10.833119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518865790316052:2059] [node 4] ICS01 socket: 402 reason# EndOfStream
2025-11-11T17:20:10.833159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518861495348689:2059] [node 4] ICP30 unregister session Session# [6:7571518865790316052:2059] VirtualId# [6:7571518861495348751:0]
2025-11-11T17:20:10.833190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518865790316052:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.833262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:10.833300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:10.889731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:10.889950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518865790316066:2061] [node 5] ICS01 socket: 433 reason# EndOfStream
2025-11-11T17:20:10.889993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518861495348690:2061] [node 5] ICP30 unregister session Session# [6:7571518865790316066:2061] VirtualId# [6:7571518865790316057:0]
2025-11-11T17:20:10.890021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518865790316066:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:10.890093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:10.890139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:11.060016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518867386937974:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:11.060056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518867386937975:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:11.133640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518867625866862:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:11.141057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518867625866863:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:11.185876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518866392023260:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:11.185928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518866392023261:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:11.254931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518868358008888:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:11.254977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518868358008889:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:11.255000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518868358008890:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:11.297994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518869047730454:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:11.298043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518869047730455:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:11.298079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518869047730456:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:11.473846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518870028587614:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:11.473890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518870028587615:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:11.473919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518870028587616:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:11.690114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518867386938034:2093] [node 4] ICP01 ready to work
2025-11-11T17:20:11.779606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518866392023299:2068] [node 5] ICP01 ready to work
2025-11-11T17:20:11.690436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518867386938035:2094] [node 6] ICP01 ready to work
2025-11-11T17:20:11.801691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518867625866909:2084] [node 4] ICP01 ready to work
2025-11-11T17:20:11.835533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518868358008888:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(166241586ns) {Configure:104(166236729ns) {ProcessConfigured:135(76789ns) {StartInitialHandshake:157(72737ns) {DropHandshakes:469(3272ns) {DropIncomingHandshake:441(910ns) DropOutgoingHandshake:456(909ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:476(1532ns) {SwitchToState:178(467ns)}}}}}
2025-11-11T17:20:11.781704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518867386938038:2095] [node 5] ICP01 ready to work
2025-11-11T17:20:11.929831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518870028587614:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(204306359ns) {Configure:104(204301792ns) {ProcessConfigured:135(71690ns) {StartInitialHandshake:157(68872ns) {DropHandshakes:469(2397ns) {DropIncomingHandshake:441(476ns) DropOutgoingHandshake:456(425ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(1915ns) {SwitchToState:178(739ns)}}}}}
2025-11-11T17:20:11.930311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518870028587670:2063] [node 4] ICP01 ready to work
2025-11-11T17:20:11.930605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518870028587671:2066] [node 5] ICP01 ready to work
2025-11-11T17:20:11.907427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518869047730456:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(185395957ns) {Configure:104(185390669ns) {ProcessConfigured:135(67934ns) {StartInitialHandshake:157(65494ns) {DropHandshakes:469(2667ns) {DropIncomingHandshake:441(511ns) DropOutgoingHandshake:456(458ns)} GenerateSessionVirtualId:988(860ns) PrepareNewSessionHandshake:476(1576ns) {SwitchToState:178(619ns)}}}}}
2025-11-11T17:20:11.901635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518866392023304:2069] [node 6] ICP01 ready to work
2025-11-11T17:20:11.836019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518868358008924:2065] [node 5] ICP01 ready to work
2025-11-11T17:20:11.909245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518869047730501:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:11.909540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518869047730502:2076] [node 4] ICP01 ready to work
2025-11-11T17:20:11.855706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518867625866863:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(221618421ns) {Configure:104(221614003ns) {ProcessConfigured:135(71763ns) {StartInitialHandshake:157(68712ns) {DropHandshakes:469(2694ns) {DropIncomingHandshake:441(937ns) DropOutgoingHandshake:456(493ns)} GenerateSessionVirtualId:988(975ns) PrepareNewSessionHandshake:476(1688ns) {SwitchToState:178(884ns)}}}}}
2025-11-11T17:20:11.929816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518866392023322:2070] [node 4] ICP01 ready to work
2025-11-11T17:20:11.955564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518866392023261:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25480276ns) {Configure:104(25475784ns) {ProcessConfigured:135(59185ns) {StartInitialHandshake:157(56596ns) {DropHandshakes:469(2103ns) {DropIncomingHandshake:441(575ns) DropOutgoingHandshake:456(386ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:476(6457ns) {SwitchToState:178(519ns)}}}}}
2025-11-11T17:20:11.879177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518868358008890:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(154667742ns) {Configure:104(154663451ns) {ProcessConfigured:135(38184ns) {StartInitialHandshake:157(35560ns) {DropHandshakes:469(1048ns) {DropIncomingHandshake:441(245ns) DropOutgoingHandshake:456(216ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:476(811ns) {SwitchToState:178(349ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:11.990317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518870028587615:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(252118463ns) {Configure:104(252113456ns) {ProcessConfigured:135(79963ns) {StartInitialHandshake:157(75707ns) {DropHandshakes:469(2140ns) {DropIncomingHandshake:441(586ns) DropOutgoingHandshake:456(485ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:476(2011ns) {SwitchToState:178(850ns)}}}}}
2025-11-11T17:20:11.992711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518870028587687:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:11.992783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518870028587688:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:11.971489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518867386937974:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(169557823ns) {Configure:104(169546816ns) {ProcessConfigured:135(56407ns) {StartInitialHandshake:157(51891ns) {DropHandshakes:469(2474ns) {DropIncomingHandshake:441(549ns) DropOutgoingHandshake:456(414ns)} GenerateSessionVirtualId:988(1368ns) PrepareNewSessionHandshake:476(1629ns) {SwitchToState:178(491ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:11.856150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518867625866923:2093] [node 6] ICP01 ready to work
2025-11-11T17:20:12.003822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518867386937975:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30736341ns) {Configure:104(30730349ns) {ProcessConfigured:135(62046ns) {StartInitialHandshake:157(58381ns) {DropHandshakes:469(2251ns) {DropIncomingHandshake:441(620ns) DropOutgoingHandshake:456(528ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:476(1506ns) {SwitchToState:178(314ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:11.937839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518867625866934:2085] [node 5] ICP01 ready to work
2025-11-11T17:20:11.881019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518868358008946:2085] [node 6] ICP01 ready to work
2025-11-11T17:20:12.014098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518867386938034:2093] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(216861248ns) {Configure:104(216856171ns) {ProcessConfigured:135(4185926ns) {StartInitialHandshake:157(4182907ns) {DropHandshakes:469(2250ns) {DropIncomingHandshake:441(327ns) DropOutgoingHandshake:456(418ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:476(2960ns) {SwitchToState:178(1463ns)}}}}}
2025-11-11T17:20:11.978517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518866392023304:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21336780ns) {Configure:104(21330891ns) {ProcessConfigured:135(45781ns) {StartInitialHandshake:157(43119ns) {DropHandshakes:469(1807ns) {DropIncomingHandshake:441(432ns) DropOutgoingHandshake:456(277ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:476(1195ns) {SwitchToState:178(270ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:11.998619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518868358008946:2085] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(117086082ns) {Configure:104(117080706ns) {ProcessConfigured:135(66462ns) {StartInitialHandshake:157(63531ns) {DropHandshakes:469(2126ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(428ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(1137ns) {SwitchToState:178(514ns)}}}}}
2025-11-11T17:20:12.021781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518870028587670:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(61724034ns) {Configure:104(61720056ns) {ProcessConfigured:135(41053640ns) {StartInitialHandshake:157(41049679ns) {DropHandshakes:469(2659ns) {DropIncomingHandshake:441(488ns) DropOutgoingHandshake:456(478ns)} GenerateSessionVirtualId:988(647ns) PrepareNewSessionHandshake:476(3313ns) {SwitchToState:178(530ns)}}}}}
2025-11-11T17:20:12.011646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518866392023322:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45699331ns) {Configure:104(45693246ns) {ProcessConfigured:135(67557ns) {StartInitialHandshake:157(64644ns) {DropHandshakes:469(1254ns) {DropIncomingHandshake:441(322ns) DropOutgoingHandshake:456(225ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:476(1259ns) {SwitchToState:178(457ns)}}}}}
2025-11-11T17:20:11.999306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518868358008966:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
2025-11-11T17:20:12.014921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518871681905347:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15925
2025-11-11T17:20:12.051640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518870686990635:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22536
2025-11-11T17:20:12.077979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518869047730455:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(357131298ns) {Configure:104(357125801ns) {ProcessConfigured:135(25102084ns) {StartInitialHandshake:157(25098406ns) {DropHandshakes:469(1202ns) {DropIncomingHandshake:441(259ns) DropOutgoingHandshake:456(242ns)} GenerateSessionVirtualId:988(399ns) PrepareNewSessionHandshake:476(3909ns) {SwitchToState:178(1914ns)}}}}}
2025-11-11T17:20:12.014977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518871681905348:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15925
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.051704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518870686990636:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22536
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.074016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518870028587688:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.051757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518870686990637:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22536
2025-11-11T17:20:12.051805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518870686990638:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22536
2025-11-11T17:20:12.069867Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518870686990638:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.070320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518866392023260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.085715Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518870686990636:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.114201Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518870686990637:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.117669Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518866392023331:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.117715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518866392023331:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.130783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518870686990640:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22536
2025-11-11T17:20:12.130837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518870686990641:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22536
2025-11-11T17:20:12.070689Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518871681905348:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.086595Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518871681905347:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.125690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518867386938046:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.125731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518867386938046:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.074256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518870028587616:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.157918Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518866392023331:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:12.145422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323554991:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:12.149475Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518870028587687:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.149704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518870028587615:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.153734Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518867386938046:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.113729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518871920834237:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42756
2025-11-11T17:20:12.137863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697821:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.162218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697822:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.201798Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518873342697821:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.205891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518867386937975:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.205967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:12.206018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518867386937975:2049] [node 3] ICP22 created new session: [1:7571518871681905355:2049]
2025-11-11T17:20:12.206064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518871681905355:2049] [node 3] ICS09 handshake done sender: [1:7571518867386938046:2113] self: [1:7571518867386938045:0] peer: [3:7571518870686990639:0] socket: 419 qp: -1
2025-11-11T17:20:12.206099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518871681905355:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:12.206142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.174076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323554993:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:12.181664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518868358008966:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.113920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518871920834238:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42756
2025-11-11T17:20:12.185809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518866392023322:2070] [node 4] ICP35 event processing took too much time PendingConnection:278(27831155ns) {IncomingHandshake:261(8029801ns) {DropIncomingHandshake:441(799ns) PrepareNewSessionHandshake:476(1843ns) {SwitchToState:178(556ns)} GenerateSessionVirtualId:988(745ns) IssueIncomingHandshakeReply:193(11777ns)}}
2025-11-11T17:20:12.174145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323554994:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:12.174509Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518870028587687:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:12.174715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518870028587615:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.174761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:12.174804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518870028587615:2049] [node 2] ICP22 created new session: [6:7571518874323554995:2049]
2025-11-11T17:20:12.174859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518874323554995:2049] [node 2] ICS09 handshake done sender: [6:7571518870028587687:2113] self: [6:7571518874323554992:0] peer: [2:7571518867625866929:0] socket: 409 qp: -1
2025-11-11T17:20:12.174889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518874323554995:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:12.174921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:12.194042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518870028587688:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:12.199014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518870028587616:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.199082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:12.199123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518870028587616:2050] [node 3] ICP22 created new session: [6:7571518874323554998:2050]
2025-11-11T17:20:12.199166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518874323554998:2050] [node 3] ICS09 handshake done sender: [6:7571518870028587688:2114] self: [6:7571518874323554990:0] peer: [3:7571518866392023330:0] socket: 410 qp: -1
2025-11-11T17:20:12.199196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518874323554998:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:12.199225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:12.197944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518866392023304:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.198013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:12.198064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518866392023304:2069] [node 6] ICP22 created new session: [3:7571518870686990645:2069]
2025-11-11T17:20:12.198110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518870686990645:2069] [node 6] ICS09 handshake done sender: [3:7571518866392023331:2112] self: [3:7571518866392023330:0] peer: [6:7571518874323554990:0] socket: 408 qp: -1
2025-11-11T17:20:12.198145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518870686990645:2069] [node 6] ICS10 traffic start
2025-11-11T17:20:12.198179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:12.218593Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:12.219429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518866392023260:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518870686990638:2118] held: no
2025-11-11T17:20:12.234442Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518870686990638:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:12.253747Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518874323554993:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.114016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518871920834239:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42756
2025-11-11T17:20:12.215569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518872652976264:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.249862Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518870686990635:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.229345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518872652976265:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
2025-11-11T17:20:12.142291Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518871920834237:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.274842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518873342697822:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.275500Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518873342697824:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.249949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518866392023260:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.275520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518873342697824:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.269772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518870028587671:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.142382Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518871920834239:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.274678Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518867386938042:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.249998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:12.229735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518868358008889:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.144844Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518871920834238:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.276028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518869047730502:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.333951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323555002:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:12.334432Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518874323554993:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:12.274750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518867386938042:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.334664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518870028587671:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.334714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:12.341698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518870028587671:2066] [node 5] ICP22 created new session: [6:7571518874323555003:2066]
2025-11-11T17:20:12.302143Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518867386938044:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.302169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518867386938044:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.303120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518871681905358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15925
2025-11-11T17:20:12.306131Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518867386938042:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:12.313527Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518867386938044:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:12.357935Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518871681905358:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.358116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518867386937974:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.358158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:12.358210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518867386937974:2048] [node 2] ICP22 created new session: [1:7571518871681905362:2048]
2025-11-11T17:20:12.358260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518871681905362:2048] [node 2] ICS09 handshake done sender: [1:7571518867386938042:2111] self: [1:7571518867386938041:0] peer: [2:7571518871920834249:0] socket: 423 qp: -1
2025-11-11T17:20:12.358301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518871681905362:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:12.358335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:12.358884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518867386938038:2095] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.358911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:12.358949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518867386938038:2095] [node 5] ICP22 created new session: [1:7571518871681905364:2095]
2025-11-11T17:20:12.358993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518871681905364:2095] [node 5] ICS09 handshake done sender: [1:7571518867386938044:2112] self: [1:7571518867386938043:0] peer: [5:7571518873342697833:0] socket: 418 qp: -1
2025-11-11T17:20:12.359022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518871681905364:2095] [node 5] ICS10 traffic start
2025-11-11T17:20:12.359047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:12.245004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518868358008889:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20343343ns) {IncomingHandshake:261(20339983ns) {DropIncomingHandshake:441(358ns) PrepareNewSessionHandshake:476(2142ns) {SwitchToState:178(989ns)} GenerateSessionVirtualId:988(654ns) IssueIncomingHandshakeReply:193(4072643ns) {SwitchToState:178(1493ns)}}}
2025-11-11T17:20:12.276366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697830:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.250046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518866392023260:2048] [node 1] ICP22 created new session: [3:7571518870686990648:2048]
2025-11-11T17:20:12.359483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518874323555003:2066] [node 5] ICS09 handshake done sender: [6:7571518874323554993:2118] self: [6:7571518874323555001:0] peer: [5:7571518873342697823:0] socket: 440 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.373702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518874323555003:2066] [node 5] ICS10 traffic start
2025-11-11T17:20:12.373773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:12.374256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518870028587671:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(39362466ns) {HandleHandshakeStatus:329(39360567ns) {DropIncomingHandshake:441(907ns) DropOutgoingHandshake:456(3914ns) SwitchToState:178(439ns) ProcessPendingSessionEvents:495(109744ns) {ForwardSessionEventToSession:612(57991ns) ForwardSessionEventToSession:612(5232ns) ForwardSessionEventToSession:612(4804ns) ForwardSessionEventToSession:612(5766ns) ForwardSessionEventToSession:612(5561ns) ForwardSessionEventToSession:612(2287ns) ForwardSessionEventToSession:612(3234ns) ForwardSessionEventToSession:612(5359ns) ForwardSessionEventToSession:612(2781ns) ForwardSessionEventToSession:612(9164ns)}}}
2025-11-11T17:20:12.374858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323555004:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:12.374941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323555005:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.157812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518871920834240:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.250086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518870686990648:2048] [node 1] ICS09 handshake done sender: [3:7571518870686990638:2118] self: [3:7571518870686990639:0] peer: [1:7571518867386938045:0] socket: 427 qp: -1
2025-11-11T17:20:12.276475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518869047730454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.378990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518867386937974:2048] [node 2] ICP18 handshake (actor [1:7571518871681905358:2118]) is too old
2025-11-11T17:20:12.169774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518867625866930:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.290071Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518872652976264:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.379177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518871681905361:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15925
2025-11-11T17:20:12.406491Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518871681905361:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.283456Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518873342697824:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:12.418159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518867386938048:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.418193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518867386938048:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.419293Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518867386938048:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:12.310634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697834:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.310744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697835:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.310823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697836:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.310912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697837:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.365750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518873342697834:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.405799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518869047730501:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.393848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518874323555005:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.419625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518867386938035:2094] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.290270Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:12.250123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518870686990648:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:12.425785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518873342697830:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.433871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:12.433927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518869047730501:2067] [node 6] ICP22 created new session: [5:7571518873342697838:2067]
2025-11-11T17:20:12.433978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518873342697838:2067] [node 6] ICS09 handshake done sender: [5:7571518873342697824:2115] self: [5:7571518873342697823:0] peer: [6:7571518874323555001:0] socket: 439 qp: -1
2025-11-11T17:20:12.434015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518873342697838:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:12.434078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:12.434575Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518869047730501:2067] [node 6] ICP35 event processing took too much time PendingConnection:278(28542437ns) {HandleHandshakeStatus:329(28539353ns) {DropIncomingHandshake:441(457ns) DropOutgoingHandshake:456(2745ns) SwitchToState:178(291ns) ProcessPendingSessionEvents:495(92361ns) {ForwardSessionEventToSession:612(45237ns) ForwardSessionEventToSession:612(3765ns) ForwardSessionEventToSession:612(5890ns) ForwardSessionEventToSession:612(2065ns) ForwardSessionEventToSession:612(3720ns) ForwardSessionEventToSession:612(7136ns) ForwardSessionEventToSession:612(2749ns) ForwardSessionEventToSession:612(5052ns) ForwardSessionEventToSession:612(2452ns) ForwardSessionEventToSession:612(2247ns)}}}
2025-11-11T17:20:12.434955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518869047730501:2067] [node 6] ICP18 handshake (actor [5:7571518873342697830:2117]) is too old
2025-11-11T17:20:12.435301Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518873342697822:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:12.438222Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518873342697835:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.169807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518867625866930:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.446446Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518873342697821:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:12.394005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518870028587670:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.460254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518869047730455:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.250158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:12.250924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518866392023261:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.262146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518870686990652:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22536
2025-11-11T17:20:12.282701Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:12.298114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518870686990640:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.298223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518866392023261:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518870686990635:2115] held: no
2025-11-11T17:20:12.298370Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518870686990635:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:12.335015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518866392023261:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.335086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:12.335135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518866392023261:2049] [node 2] ICP22 created new session: [3:7571518870686990653:2049]
2025-11-11T17:20:12.335202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518870686990653:2049] [node 2] ICS09 handshake done sender: [3:7571518870686990635:2115] self: [3:7571518870686990651:0] peer: [2:7571518867625866921:0] socket: 421 qp: -1
2025-11-11T17:20:12.335238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518870686990653:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:12.335278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:12.366664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518870686990634:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.366706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518870686990634:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.385826Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518870686990634:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:12.406226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518866392023322:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.406284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:12.425923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518866392023322:2070] [node 4] ICP22 created new session: [3:7571518870686990657:2070]
2025-11-11T17:20:12.438414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518870686990657:2070] [node 4] ICS09 handshake done sender: [3:7571518870686990634:2114] self: [3:7571518870686990633:0] peer: [4:7571518872652976275:0] socket: 445 qp: -1
2025-11-11T17:20:12.446556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518870686990657:2070] [node 4] ICS10 traffic start
2025-11-11T17:20:12.446605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:12.290898Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518868358008945:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.290920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518868358008945:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.291459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518868358008889:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518868358008966:2115] held: no
2025-11-11T17:20:12.291748Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518868358008966:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:12.306617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518868358008889:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.306693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:12.306755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518868358008889:2049] [node 2] ICP22 created new session: [4:7571518872652976268:2049]
2025-11-11T17:20:12.306807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518872652976268:2049] [node 2] ICS09 handshake done sender: [4:7571518868358008966:2115] self: [4:7571518872652976263:0] peer: [2:7571518867625866925:0] socket: 413 qp: -1
2025-11-11T17:20:12.306840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518872652976268:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:12.306876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:12.307323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518872652976266:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
2025-11-11T17:20:12.311220Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518868358008945:2099] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:12.311484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518868358008924:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.311543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:12.311599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518868358008924:2065] [node 5] ICP22 created new session: [4:7571518872652976272:2065]
2025-11-11T17:20:12.311642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518872652976272:2065] [node 5] ICS09 handshake done sender: [4:7571518868358008945:2099] self: [4:7571518868358008944:0] peer: [5:7571518873342697832:0] socket: 415 qp: -1
2025-11-11T17:20:12.311674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518872652976272:2065] [node 5] ICS10 traffic start
2025-11-11T17:20:12.311710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:12.336590Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518872652976266:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.336718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518868358008890:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.390751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518872652976276:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
2025-11-11T17:20:12.413255Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518868358008965:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.413279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518868358008965:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.414066Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518872652976266:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:12.419765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518872652976277:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
2025-11-11T17:20:12.437958Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518868358008965:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:12.438493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518868358008890:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.473556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:12.173866Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518867625866930:2109] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:12.449728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:12.449802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518867386938035:2094] [node 6] ICP22 created new session: [1:7571518871681905369:2094]
2025-11-11T17:20:12.449855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518871681905369:2094] [node 6] ICS09 handshake done sender: [1:7571518867386938048:2114] self: [1:7571518867386938047:0] peer: [6:7571518874323555008:0] socket: 454 qp: -1
2025-11-11T17:20:12.449892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518871681905369:2094] [node 6] ICS10 traffic start
2025-11-11T17:20:12.449946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:12.450403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518867386938035:2094] [node 6] ICP35 event processing took too much time PendingConnection:278(30558534ns) {HandleHandshakeStatus:329(30556541ns) {DropIncomingHandshake:441(40426ns) DropOutgoingHandshake:456(298ns) SwitchToState:178(345ns) ProcessPendingSessionEvents:495(100537ns) {ForwardSessionEventToSession:612(49765ns) ForwardSessionEventToSession:612(10802ns) ForwardSessionEventToSession:612(4263ns) ForwardSessionEventToSession:612(3788ns) ForwardSessionEventToSession:612(2281ns) ForwardSessionEventToSession:612(4431ns) ForwardSessionEventToSession:612(2277ns) ForwardSessionEventToSession:612(5224ns) ForwardSessionEventToSession:612(2439ns) ForwardSessionEventToSession:612(4572ns)}}}
2025-11-11T17:20:12.417764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518874323555004:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.492682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518866392023322:2070] [node 4] ICP35 event processing took too much time PendingConnection:278(94432980ns) {HandleHandshakeStatus:329(86208439ns) {DropIncomingHandshake:441(14756ns) DropOutgoingHandshake:456(425ns) SwitchToState:178(487ns) ProcessPendingSessionEvents:495(13288909ns) {ForwardSessionEventToSession:612(13242477ns) ForwardSessionEventToSession:612(7252ns) ForwardSessionEventToSession:612(2450ns) ForwardSessionEventToSession:612(5099ns) ForwardSessionEventToSession:612(2760ns) ForwardSessionEventToSession:612(2777ns) ForwardSessionEventToSession:612(4457ns) ForwardSessionEventToSession:612(2615ns) ForwardSessionEventToSession:612(4614ns) ForwardSessionEventToSession:612(4838ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.217800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518867625866923:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.489720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518869047730454:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.417887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518870028587614:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.479838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518868358008890:2050] [node 3] ICP22 created new session: [4:7571518872652976279:2050]
2025-11-11T17:20:12.489844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:12.489890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518869047730454:2048] [node 1] ICP22 created new session: [5:7571518873342697842:2048]
2025-11-11T17:20:12.489931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518873342697842:2048] [node 1] ICS09 handshake done sender: [5:7571518873342697822:2114] self: [5:7571518873342697833:0] peer: [1:7571518867386938043:0] socket: 431 qp: -1
2025-11-11T17:20:12.489968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518873342697842:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:12.490004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:12.490975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518869047730502:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.491049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:12.491098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518869047730502:2076] [node 4] ICP22 created new session: [5:7571518873342697844:2076]
2025-11-11T17:20:12.491135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518873342697844:2076] [node 4] ICS09 handshake done sender: [5:7571518873342697821:2113] self: [5:7571518873342697832:0] peer: [4:7571518868358008944:0] socket: 430 qp: -1
2025-11-11T17:20:12.491170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518873342697844:2076] [node 4] ICS10 traffic start
2025-11-11T17:20:12.491215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:12.492074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518869047730456:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.225801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:12.515918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518866392023333:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.498721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518871681905346:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.494694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518868358008946:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.531927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518866392023333:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.543008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697847:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.430263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323555009:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:12.225867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518867625866923:2093] [node 6] ICP22 created new session: [2:7571518871920834245:2093]
2025-11-11T17:20:12.225936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518871920834245:2093] [node 6] ICS09 handshake done sender: [2:7571518867625866930:2109] self: [2:7571518867625866929:0] peer: [6:7571518874323554992:0] socket: 404 qp: -1
2025-11-11T17:20:12.225980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518871920834245:2093] [node 6] ICS10 traffic start
2025-11-11T17:20:12.226017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:12.226961Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518871920834240:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.227941Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518867625866926:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.227961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518867625866926:2107] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.228715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518867625866862:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.235448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518871920834250:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42756
2025-11-11T17:20:12.238439Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518867625866926:2107] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:12.254332Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518867625866922:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.254355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518867625866922:2105] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.254818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518867625866909:2084] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.254894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:12.254942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518867625866909:2084] [node 4] ICP22 created new session: [2:7571518871920834251:2084]
2025-11-11T17:20:12.254986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518871920834251:2084] [node 4] ICS09 handshake done sender: [2:7571518867625866926:2107] self: [2:7571518867625866925:0] peer: [4:7571518872652976263:0] socket: 403 qp: -1
2025-11-11T17:20:12.255020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518871920834251:2084] [node 4] ICS10 traffic start
2025-11-11T17:20:12.255051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:12.256349Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518867625866922:2105] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:12.256578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518867625866863:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.256660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:12.256700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518867625866863:2049] [node 3] ICP22 created new session: [2:7571518871920834255:2049]
2025-11-11T17:20:12.256753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518871920834255:2049] [node 3] ICS09 handshake done sender: [2:7571518867625866922:2105] self: [2:7571518867625866921:0] peer: [3:7571518870686990651:0] socket: 402 qp: -1
2025-11-11T17:20:12.256787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518871920834255:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:12.256818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:12.301773Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518871920834250:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.309971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518871920834258:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42756
2025-11-11T17:20:12.312410Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518871920834240:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:12.312598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518867625866862:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.312661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:12.312707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518867625866862:2048] [node 1] ICP22 created new session: [2:7571518871920834259:2048]
2025-11-11T17:20:12.312747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518871920834259:2048] [node 1] ICS09 handshake done sender: [2:7571518871920834240:2118] self: [2:7571518871920834249:0] peer: [1:7571518867386938041:0] socket: 429 qp: -1
2025-11-11T17:20:12.312773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518871920834259:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:12.312801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:12.493301Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518867625866937:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.493324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518867625866937:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.498756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518871681905346:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:12.494787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:12.473625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518874323555010:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43516
2025-11-11T17:20:12.475042Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:12.500075Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518874323555004:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:12.500451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518870028587614:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518874323555004:2121] held: no
2025-11-11T17:20:12.500544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518870028587614:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.500583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:12.500625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518870028587614:2048] [node 1] ICP22 created new session: [6:7571518874323555011:2048]
2025-11-11T17:20:12.500664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518874323555011:2048] [node 1] ICS09 handshake done sender: [6:7571518874323555004:2121] self: [6:7571518874323555008:0] peer: [1:7571518867386938047:0] socket: 447 qp: -1
2025-11-11T17:20:12.500693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518874323555011:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:12.500723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:12.501096Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18559) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:12.514337Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518874323555005:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:12.514556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518870028587670:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18559) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518874323555005:2122] held: no
2025-11-11T17:20:12.514641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518870028587670:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.514682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:12.514746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518870028587670:2063] [node 4] ICP22 created new session: [6:7571518874323555013:2063]
2025-11-11T17:20:12.514797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518874323555013:2063] [node 4] ICS09 handshake done sender: [6:7571518874323555005:2122] self: [6:7571518874323555007:0] peer: [4:7571518868358008964:0] socket: 461 qp: -1
2025-11-11T17:20:12.514832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518874323555013:2063] [node 4] ICS10 traffic start
2025-11-11T17:20:12.514869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:12.506114Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518871681905346:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:12.533874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518867386938034:2093] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.533960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:12.534034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518867386938034:2093] [node 4] ICP22 created new session: [1:7571518871681905372:2093]
2025-11-11T17:20:12.534080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518871681905372:2093] [node 4] ICS09 handshake done sender: [1:7571518871681905346:2115] self: [1:7571518871681905345:0] peer: [4:7571518872652976283:0] socket: 465 qp: -1
2025-11-11T17:20:12.534113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518871681905372:2093] [node 4] ICS10 traffic start
2025-11-11T17:20:12.534153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:12.560477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518867386938034:2093] [node 4] ICP35 event processing took too much time PendingConnection:278(26368111ns) {HandleHandshakeStatus:329(26365081ns) {DropIncomingHandshake:441(14838ns) DropOutgoingHandshake:456(631ns) SwitchToState:178(582ns) ProcessPendingSessionEvents:495(25936307ns) {ForwardSessionEventToSession:612(25870820ns) ForwardSessionEventToSession:612(12380ns) ForwardSessionEventToSession:612(3467ns) ForwardSessionEventToSession:612(6306ns) ForwardSessionEventToSession:612(2379ns) ForwardSessionEventToSession:612(5407ns) ForwardSessionEventToSession:612(3012ns) ForwardSessionEventToSession:612(11153ns) ForwardSessionEventToSession:612(6024ns) ForwardSessionEventToSession:612(2689ns)}}}
2025-11-11T17:20:12.494916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518868358008946:2085] [node 6] ICP22 created new session: [4:7571518872652976280:2085]
2025-11-11T17:20:12.561787Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518867625866937:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:12.573937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518867625866934:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.572012Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518873342697834:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:12.494959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518872652976280:2085] [node 6] ICS09 handshake done sender: [4:7571518868358008965:2114] self: [4:7571518868358008964:0] peer: [6:7571518874323555007:0] socket: 460 qp: -1
2025-11-11T17:20:12.494997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518872652976280:2085] [node 6] ICS10 traffic start
2025-11-11T17:20:12.495050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:12.496412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518872652976282:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
2025-11-11T17:20:12.498385Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518872652976277:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:12.498497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518868358008888:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:12.500215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518872652976284:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32584
2025-11-11T17:20:12.515448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518872652976279:2050] [node 3] ICS09 handshake done sender: [4:7571518872652976266:2118] self: [4:7571518872652976275:0] peer: [3:7571518870686990633:0] socket: 452 qp: -1
2025-11-11T17:20:12.525857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518872652976279:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:12.525929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:12.561266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518868358008890:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(122479342ns) {HandleHandshakeStatus:329(122475925ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(6091ns) SwitchToState:178(564ns) ProcessPendingSessionEvents:495(11133822ns) {ForwardSessionEventToSession:612(11073504ns) ForwardSessionEventToSession:612(9690ns) ForwardSessionEventToSession:612(3557ns) ForwardSessionEventToSession:612(11915ns) ForwardSessionEventToSession:612(3455ns) ForwardSessionEventToSession:612(2297ns) ForwardSessionEventToSession:612(5841ns) ForwardSessionEventToSession:612(2040ns) ForwardSessionEventToSession:612(1672ns) ForwardSessionEventToSession:612(4183ns)}}}
2025-11-11T17:20:12.561373Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:20:12.570598Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:12.577922Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518866392023333:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:12.594600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518866392023299:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:12.594690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:12.594736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518866392023299:2068] [node 5] ICP22 created new session: [3:7571518870686990660:2068]
2025-11-11T17:20:12.594778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518870686990660:2068] [node 5] ICS09 handshake done sender: [3:7571518866392023333:2113] self: [3:7571518866392023332:0] peer: [5:7571518873342697846:0] socket: 428 qp: -1
2025-11-11T17:20:12.594814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518870686990660:2068] [node 5] ICS10 traffic start
2025-11-11T17:20:12.594869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:12.597816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518872652976277:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:12.617699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:12.617799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518867625866934:2085] [node 5] ICP22 created new session: [2:7571518871920834262:2085]
2025-11-11T17:20:12.617855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518871920834262:2085] [node 5] ICS09 handshake done sender: [2:7571518867625866937:2114] self: [2:7571518867625866936:0] peer: [5:7571518873342697840:0] socket: 407 qp: -1
2025-11-11T17:20:12.617904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518871920834262:2085] [node 5] ICS10 traffic start
2025-11-11T17:20:12.617972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:12.618510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518867625866934:2085] [node 5] ICP35 event processing took too much time PendingConnection:278(44350215ns) {HandleHandshakeStatus:329(44347636ns) {DropIncomingHandshake:441(24619ns) DropOutgoingHandshake:456(467ns) SwitchToState:178(477ns) ProcessPendingSessionEvents:495(110976ns) {ForwardSessionEventToSession:612(59046ns) ForwardSessionEventToSession:612(9137ns) ForwardSessionEventToSession:612(2672ns) ForwardSessionEventToSession:612(3123ns) ForwardSessionEventToSession:612(5926ns) ForwardSessionEventToSession:612(2444ns) ForwardSessionEventToSession:612(2547ns) ForwardSessionEventToSession:612(10279ns) ForwardSessionEventToSession:612(3147ns) ForwardSessionEventToSession:612(5214ns)}}}
2025-11-11T17:20:12.590401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518869047730455:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.590509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:12.590569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518869047730455:2049] [node 2] ICP22 created new session: [5:7571518873342697849:2049]
2025-11-11T17:20:12.590625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518873342697849:2049] [node 2] ICS09 handshake done sender: [5:7571518873342697834:2118] self: [5:7571518873342697840:0] peer: [2:7571518867625866936:0] socket: 424 qp: -1
2025-11-11T17:20:12.590661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518873342697849:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:12.590694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:12.593139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518873342697848:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25610
2025-11-11T17:20:12.593531Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518873342697835:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:12.597996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518869047730456:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.598068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:12.598115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518869047730456:2050] [node 3] ICP22 created new session: [5:7571518873342697851:2050]
2025-11-11T17:20:12.598165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518873342697851:2050] [node 3] ICS09 handshake done sender: [5:7571518873342697835:2119] self: [5:7571518873342697846:0] peer: [3:7571518866392023332:0] socket: 453 qp: -1
2025-11-11T17:20:12.598196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518873342697851:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:12.598239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:12.598903Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2136) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:12.626308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518869047730456:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:12.633808Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518868358008888:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518872652976277:2120] held: no
2025-11-11T17:20:12.633931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518868358008888:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:12.634003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:12.634056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518868358008888:2048] [node 1] ICP22 created new session: [4:7571518872652976286:2048]
2025-11-11T17:20:12.634120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518872652976286:2048] [node 1] ICS09 handshake done sender: [4:7571518872652976277:2120] self: [4:7571518872652976283:0] peer: [1:7571518871681905345:0] socket: 466 qp: -1
2025-11-11T17:20:12.634161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518872652976286:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:12.634211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:12.689789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:12.690017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518871920834259:2048] [node 1] ICS01 socket: 429 reason# EndOfStream
2025-11-11T17:20:12.690059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518867625866862:2048] [node 1] ICP30 unregister session Session# [2:7571518871920834259:2048] VirtualId# [2:7571518871920834249:0]
2025-11-11T17:20:12.690095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518871920834259:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.690202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:12.690246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:12.701697Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:12.701808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:12.717927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518872652976286:2048] [node 1] ICS01 socket: 466 reason# EndOfStream
2025-11-11T17:20:12.717978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518868358008888:2048] [node 1] ICP30 unregister session Session# [4:7571518872652976286:2048] VirtualId# [4:7571518872652976283:0]
2025-11-11T17:20:12.718018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518872652976286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.718130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:12.718173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:12.700773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:12.701116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518870686990648:2048] [node 1] ICS01 socket: 427 reason# EndOfStream
2025-11-11T17:20:12.701158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518866392023260:2048] [node 1] ICP30 unregister session Session# [3:7571518870686990648:2048] VirtualId# [3:7571518870686990639:0]
2025-11-11T17:20:12.701193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518870686990648:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.701277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:12.701319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:12.700911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:12.720990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518873342697842:2048] [node 1] ICS01 socket: 431 reason# EndOfStream
2025-11-11T17:20:12.721035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518869047730454:2048] [node 1] ICP30 unregister session Session# [5:7571518873342697842:2048] VirtualId# [5:7571518873342697833:0]
2025-11-11T17:20:12.721066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518873342697842:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.721148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:12.721187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:12.702036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518874323555011:2048] [node 1] ICS01 socket: 447 reason# EndOfStream
2025-11-11T17:20:12.702068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518870028587614:2048] [node 1] ICP30 unregister session Session# [6:7571518874323555011:2048] VirtualId# [6:7571518874323555008:0]
2025-11-11T17:20:12.702092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518874323555011:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.702163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:12.702203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:12.825262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:12.825468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518870686990653:2049] [node 2] ICS01 socket: 421 reason# EndOfStream
2025-11-11T17:20:12.825510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518866392023261:2049] [node 2] ICP30 unregister session Session# [3:7571518870686990653:2049] VirtualId# [3:7571518870686990651:0]
2025-11-11T17:20:12.825541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518870686990653:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.825809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:12.825874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:12.829729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:12.829947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518872652976268:2049] [node 2] ICS01 socket: 413 reason# EndOfStream
2025-11-11T17:20:12.829986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518868358008889:2049] [node 2] ICP30 unregister session Session# [4:7571518872652976268:2049] VirtualId# [4:7571518872652976263:0]
2025-11-11T17:20:12.830030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518872652976268:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.830131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:12.830172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:12.830707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:12.830913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518873342697849:2049] [node 2] ICS01 socket: 424 reason# EndOfStream
2025-11-11T17:20:12.830952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518869047730455:2049] [node 2] ICP30 unregister session Session# [5:7571518873342697849:2049] VirtualId# [5:7571518873342697840:0]
2025-11-11T17:20:12.830974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518873342697849:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.831037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:12.831068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:12.833708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:12.833890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518874323554995:2049] [node 2] ICS01 socket: 409 reason# EndOfStream
2025-11-11T17:20:12.833924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518870028587615:2049] [node 2] ICP30 unregister session Session# [6:7571518874323554995:2049] VirtualId# [6:7571518874323554992:0]
2025-11-11T17:20:12.833948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518874323554995:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.834025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:12.834098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:12.953738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:12.953993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518873342697851:2050] [node 3] ICS01 socket: 453 reason# EndOfStream
2025-11-11T17:20:12.954038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518869047730456:2050] [node 3] ICP30 unregister session Session# [5:7571518873342697851:2050] VirtualId# [5:7571518873342697846:0]
2025-11-11T17:20:12.954075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518873342697851:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.954158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:12.954200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:12.954802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:12.955011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518874323554998:2050] [node 3] ICS01 socket: 410 reason# EndOfStream
2025-11-11T17:20:12.955048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518870028587616:2050] [node 3] ICP30 unregister session Session# [6:7571518874323554998:2050] VirtualId# [6:7571518874323554990:0]
2025-11-11T17:20:12.955081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518874323554998:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.955155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:12.955193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:12.957684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:12.957916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518872652976279:2050] [node 3] ICS07 socket disconnect 452 reason# EndOfStream
2025-11-11T17:20:12.957944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518872652976279:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:12.958018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:12.958061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518872652976279:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:12.962097Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2136) Couldn't connect to any resolved address
2025-11-11T17:20:12.962379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518868358008890:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2136) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:12.962424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518872652976279:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:12.962458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518868358008890:2050] [node 3] ICP30 unregister session Session# [4:7571518872652976279:2050] VirtualId# [4:7571518872652976275:0]
2025-11-11T17:20:12.962501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:12.962527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518868358008890:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:12.962565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:13.037989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:13.038196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518874323555013:2063] [node 4] ICS01 socket: 461 reason# EndOfStream
2025-11-11T17:20:13.038245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518870028587670:2063] [node 4] ICP30 unregister session Session# [6:7571518874323555013:2063] VirtualId# [6:7571518874323555007:0]
2025-11-11T17:20:13.038280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518874323555013:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:13.038360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:13.038397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:13.038166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:13.038399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518873342697844:2076] [node 4] ICS01 socket: 430 reason# EndOfStream
2025-11-11T17:20:13.038434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518869047730502:2076] [node 4] ICP30 unregister session Session# [5:7571518873342697844:2076] VirtualId# [5:7571518873342697832:0]
2025-11-11T17:20:13.038465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518873342697844:2076] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:13.038537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:13.038582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:13.157757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:13.157976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518874323555003:2066] [node 5] ICS01 socket: 440 reason# EndOfStream
2025-11-11T17:20:13.158019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518870028587671:2066] [node 5] ICP30 unregister session Session# [6:7571518874323555003:2066] VirtualId# [6:7571518874323555001:0]
2025-11-11T17:20:13.158049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518874323555003:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:13.158133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:13.158176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:13.299383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518875155691808:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:13.299424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518875155691809:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:13.327590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518876223533876:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:13.327633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518876223533877:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:13.361804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518875668012113:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:13.361845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518875668012114:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:13.423352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518876491281342:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:13.423396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518876491281343:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:13.423424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518876491281344:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:13.464856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518877019297209:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:13.464915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518877019297210:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:13.464944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518877019297211:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:13.773086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518875668012137:2071] [node 5] ICP01 ready to work
2025-11-11T17:20:13.791914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518878003255064:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:13.791950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518878003255065:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:13.791976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518878003255066:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:13.924393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518875668012114:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(134413274ns) {Configure:104(134408551ns) {ProcessConfigured:135(67659ns) {StartInitialHandshake:157(65141ns) {DropHandshakes:469(2569ns) {DropIncomingHandshake:441(583ns) DropOutgoingHandshake:456(467ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:476(1577ns) {SwitchToState:178(569ns)}}}}}
2025-11-11T17:20:13.924823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518875668012167:2083] [node 6] ICP01 ready to work
2025-11-11T17:20:13.965818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518875155691863:2085] [node 6] ICP01 ready to work
2025-11-11T17:20:13.978757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518876223533932:2084] [node 6] ICP01 ready to work
2025-11-11T17:20:14.001723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518877019297228:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.072728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518879962979481:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19985
2025-11-11T17:20:14.069946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518882298222377:2062] [node 4] ICP01 ready to work
2025-11-11T17:20:14.026079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518875155691862:2086] [node 4] ICP01 ready to work
2025-11-11T17:20:14.055242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518880786248701:2082] [node 5] ICP01 ready to work
2025-11-11T17:20:14.069674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518880518501236:2083] [node 5] ICP01 ready to work
2025-11-11T17:20:14.136852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518876223533932:2084] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(51009494ns) {Configure:104(51002451ns) {ProcessConfigured:135(77566ns) {StartInitialHandshake:157(73764ns) {DropHandshakes:469(4826ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(374ns)} GenerateSessionVirtualId:988(1328ns) PrepareNewSessionHandshake:476(1893ns) {SwitchToState:178(592ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.039967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518875155691868:2084] [node 5] ICP01 ready to work
2025-11-11T17:20:14.069674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518879962979479:2075] [node 4] ICP01 ready to work
2025-11-11T17:20:14.055612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518880786248702:2084] [node 6] ICP01 ready to work
2025-11-11T17:20:14.076327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518882298222384:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.174834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518876223533876:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(37716409ns) {Configure:104(37712214ns) {ProcessConfigured:135(92682ns) {StartInitialHandshake:157(89759ns) {DropHandshakes:469(2397ns) {DropIncomingHandshake:441(568ns) DropOutgoingHandshake:456(431ns)} GenerateSessionVirtualId:988(1003ns) PrepareNewSessionHandshake:476(1581ns) {SwitchToState:178(713ns)}}}}}
2025-11-11T17:20:14.182240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222426:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.189639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518881314264562:2063] [node 6] ICP01 ready to work
2025-11-11T17:20:14.207395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518879962979479:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(57452359ns) {Configure:104(57444948ns) {ProcessConfigured:135(54573ns) {StartInitialHandshake:157(51989ns) {DropHandshakes:469(1555ns) {DropIncomingHandshake:441(285ns) DropOutgoingHandshake:456(324ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:476(1112ns) {SwitchToState:178(381ns)}}}}}
2025-11-11T17:20:14.189821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518880518501240:2085] [node 4] ICP01 ready to work
2025-11-11T17:20:14.249809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518876491281343:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(431669723ns) {Configure:104(431665031ns) {ProcessConfigured:135(58617ns) {StartInitialHandshake:157(56079ns) {DropHandshakes:469(2815ns) {DropIncomingHandshake:441(818ns) DropOutgoingHandshake:456(381ns)} GenerateSessionVirtualId:988(1193ns) PrepareNewSessionHandshake:476(1155ns) {SwitchToState:178(561ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.266221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518879962979489:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19985
2025-11-11T17:20:14.266545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518879962979490:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19985
2025-11-11T17:20:14.198475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518876223533877:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21907815ns) {Configure:104(21903906ns) {ProcessConfigured:135(73169ns) {StartInitialHandshake:157(70655ns) {DropHandshakes:469(2320ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(339ns)} GenerateSessionVirtualId:988(843ns) PrepareNewSessionHandshake:476(1597ns) {SwitchToState:178(512ns)}}}}}
2025-11-11T17:20:14.224981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518877019297209:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(155105888ns) {Configure:104(155095607ns) {ProcessConfigured:135(79784ns) {StartInitialHandshake:157(77798ns) {DropHandshakes:469(2327ns) {DropIncomingHandshake:441(530ns) DropOutgoingHandshake:456(448ns)} GenerateSessionVirtualId:988(959ns) PrepareNewSessionHandshake:476(1694ns) {SwitchToState:178(543ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.234123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518882298222426:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.278017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264572:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.200667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518880518501237:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14854
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.310747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518875155691808:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(208829571ns) {Configure:104(208818594ns) {ProcessConfigured:135(97269ns) {StartInitialHandshake:157(93720ns) {DropHandshakes:469(1976ns) {DropIncomingHandshake:441(489ns) DropOutgoingHandshake:456(385ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:476(1665ns) {SwitchToState:178(491ns)}}}}}
2025-11-11T17:20:14.278828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518875668012137:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(464431505ns) {Configure:104(464415925ns) {ProcessConfigured:135(47841136ns) {StartInitialHandshake:157(47838014ns) {DropHandshakes:469(1916ns) {DropIncomingHandshake:441(516ns) DropOutgoingHandshake:456(363ns)} GenerateSessionVirtualId:988(647ns) PrepareNewSessionHandshake:476(8455ns) {SwitchToState:178(6070ns)}}}}}
2025-11-11T17:20:14.270513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518882298222384:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(161417041ns) {Configure:104(161412457ns) {ProcessConfigured:135(76008131ns) {StartInitialHandshake:157(67936785ns) {DropHandshakes:469(3393ns) {DropIncomingHandshake:441(1326ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(2433ns) PrepareNewSessionHandshake:476(2305ns) {SwitchToState:178(661ns)}}}}}
2025-11-11T17:20:14.282701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518880786248702:2084] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32609917ns) {Configure:104(32602881ns) {ProcessConfigured:135(57011ns) {StartInitialHandshake:157(54089ns) {DropHandshakes:469(2709ns) {DropIncomingHandshake:441(581ns) DropOutgoingHandshake:456(494ns)} GenerateSessionVirtualId:988(628ns) PrepareNewSessionHandshake:476(1729ns) {SwitchToState:178(649ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.330709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518881314264572:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.283595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248710:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.213741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518880518501247:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14854
2025-11-11T17:20:14.317721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518879962979481:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.273690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518878003255064:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(435288414ns) {Configure:104(435283974ns) {ProcessConfigured:135(47921382ns) {StartInitialHandshake:157(47918373ns) {DropHandshakes:469(2286ns) {DropIncomingHandshake:441(482ns) DropOutgoingHandshake:456(537ns)} GenerateSessionVirtualId:988(338ns) PrepareNewSessionHandshake:476(4476ns) {SwitchToState:178(2020ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.313440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518879450659170:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33784
2025-11-11T17:20:14.333642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518879962979490:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.362007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518875668012114:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.377888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518875155691809:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(337712307ns) {Configure:104(337703224ns) {ProcessConfigured:135(11754681ns) {StartInitialHandshake:157(11752523ns) {DropHandshakes:469(148189ns) {DropIncomingHandshake:441(517ns) DropOutgoingHandshake:456(461ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:476(4548ns) {SwitchToState:178(2111ns)}}}}}
2025-11-11T17:20:14.378128Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659170:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.213903Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501237:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.343106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518877019297210:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.333735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518880786248701:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(256407232ns) {Configure:104(256400864ns) {ProcessConfigured:135(15979274ns) {StartInitialHandshake:157(15976824ns) {DropHandshakes:469(2398ns) {DropIncomingHandshake:441(550ns) DropOutgoingHandshake:456(459ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:476(3093ns) {SwitchToState:178(1591ns)}}}}}
2025-11-11T17:20:14.350002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518877019297228:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(124789488ns) {Configure:104(124785351ns) {ProcessConfigured:135(52704ns) {StartInitialHandshake:157(50872ns) {DropHandshakes:469(1698ns) {DropIncomingHandshake:441(263ns) DropOutgoingHandshake:456(355ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:476(812ns) {SwitchToState:178(382ns)}}}}}
2025-11-11T17:20:14.385799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518877019297211:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(532033159ns) {Configure:104(532022600ns) {ProcessConfigured:135(35695512ns) {StartInitialHandshake:157(35692684ns) {DropHandshakes:469(2659ns) {DropIncomingHandshake:441(540ns) DropOutgoingHandshake:456(569ns)} GenerateSessionVirtualId:988(1001ns) PrepareNewSessionHandshake:476(4081ns) {SwitchToState:178(2439ns)}}}}}
2025-11-11T17:20:14.373624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518880518501254:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14854
2025-11-11T17:20:14.408014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501244:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.401638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518879450659177:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33784
2025-11-11T17:20:14.408050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518880518501244:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.410002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518875668012171:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.410039Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518875668012171:2104] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.429777Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518875668012171:2104] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:14.395786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518878003255066:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.396943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518876491281342:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(475025922ns) {Configure:104(475014884ns) {ProcessConfigured:135(27071252ns) {StartInitialHandshake:157(27068682ns) {DropHandshakes:469(1588ns) {DropIncomingHandshake:441(503ns) DropOutgoingHandshake:456(282ns)} GenerateSessionVirtualId:988(516ns) PrepareNewSessionHandshake:476(2777ns) {SwitchToState:178(1352ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.442036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264586:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.397060Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518880786248710:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.445792Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518880786248709:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.445829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518880786248709:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.396070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222439:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.396175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222440:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.396317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222441:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.396404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222442:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.405824Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518882298222441:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.405969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518882298222377:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.437907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518882298222440:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.442430Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518882298222439:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.442517Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518882298222442:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.442626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518882298222384:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.442830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518878003255065:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.442877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518878003255064:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.422589Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659177:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.421648Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501254:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.426928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518880518501256:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14854
2025-11-11T17:20:14.445809Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518880518501244:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:14.449622Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501246:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.449658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518880518501246:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.450268Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501247:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.450361Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501239:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.450378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518880518501239:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.469627Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518881314264574:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.469681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518881314264574:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.473817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264585:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.445898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518875155691862:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(168263766ns) {Configure:104(168258015ns) {ProcessConfigured:135(49996ns) {StartInitialHandshake:157(47964ns) {DropHandshakes:469(603ns) {DropIncomingHandshake:441(173ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:476(1185ns) {SwitchToState:178(364ns)}}}}}
2025-11-11T17:20:14.466004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222449:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.493886Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518880518501246:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.498025Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518880518501239:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:14.505969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518879962979494:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19985
2025-11-11T17:20:14.498400Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501256:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.466184Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659176:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.529750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518881314264585:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.529874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518881314264572:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:14.530063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518875668012167:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.530122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:14.530191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518875668012167:2083] [node 6] ICP22 created new session: [3:7571518879962979495:2083]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.530239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518879962979495:2083] [node 6] ICS09 handshake done sender: [3:7571518875668012171:2104] self: [3:7571518875668012170:0] peer: [6:7571518882298222443:0] socket: 414 qp: -1
2025-11-11T17:20:14.530273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518879962979495:2083] [node 6] ICS10 traffic start
2025-11-11T17:20:14.530310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:14.530795Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518879962979489:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.531113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518875668012167:2083] [node 6] ICP18 handshake (actor [3:7571518879962979489:2116]) is too old
2025-11-11T17:20:14.538455Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:14.545901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518875668012114:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518879962979490:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.534037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248719:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.466219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518879450659176:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.534302Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518881314264574:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:14.498453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518880518501236:2083] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.569856Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.485740Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518879450659176:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:14.498510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:14.558074Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518879962979490:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:14.578203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518877019297210:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.553787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659172:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.534099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248720:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.498555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518880518501236:2083] [node 5] ICP22 created new session: [2:7571518880518501262:2083]
2025-11-11T17:20:14.498601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518880518501262:2083] [node 5] ICS09 handshake done sender: [2:7571518880518501244:2113] self: [2:7571518880518501243:0] peer: [5:7571518881314264581:0] socket: 421 qp: -1
2025-11-11T17:20:14.498632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518880518501262:2083] [node 5] ICS10 traffic start
2025-11-11T17:20:14.498672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:14.499164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518876223533877:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.499231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:14.499273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518876223533877:2049] [node 3] ICP22 created new session: [2:7571518880518501264:2049]
2025-11-11T17:20:14.499319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518880518501264:2049] [node 3] ICS09 handshake done sender: [2:7571518880518501246:2114] self: [2:7571518880518501245:0] peer: [3:7571518879962979492:0] socket: 423 qp: -1
2025-11-11T17:20:14.499351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518880518501264:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:14.499397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:14.500223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518876223533932:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.500258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:14.500293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518876223533932:2084] [node 6] ICP22 created new session: [2:7571518880518501266:2084]
2025-11-11T17:20:14.500335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518880518501266:2084] [node 6] ICS09 handshake done sender: [2:7571518880518501239:2111] self: [2:7571518880518501238:0] peer: [6:7571518882298222446:0] socket: 420 qp: -1
2025-11-11T17:20:14.500366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518880518501266:2084] [node 6] ICS10 traffic start
2025-11-11T17:20:14.500389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:14.501763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518876223533876:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.510421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518880518501255:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14854
2025-11-11T17:20:14.518788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501255:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.555820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518880518501272:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14854
2025-11-11T17:20:14.585749Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518880518501256:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:14.609698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518876223533876:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.609747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:14.577649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518875668012114:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.617766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518876223533876:2048] [node 1] ICP22 created new session: [2:7571518880518501273:2048]
2025-11-11T17:20:14.617845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518880518501273:2048] [node 1] ICS09 handshake done sender: [2:7571518880518501256:2119] self: [2:7571518880518501270:0] peer: [1:7571518879450659171:0] socket: 444 qp: -1
2025-11-11T17:20:14.617889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518880518501273:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:14.617924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.593675Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.534369Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518880786248709:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.578271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:14.553815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518879450659172:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.577704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:14.566246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518875155691863:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.566336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:14.566407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518875155691863:2085] [node 6] ICP22 created new session: [1:7571518879450659189:2085]
2025-11-11T17:20:14.566453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518879450659189:2085] [node 6] ICS09 handshake done sender: [1:7571518879450659176:2113] self: [1:7571518879450659175:0] peer: [6:7571518882298222447:0] socket: 440 qp: -1
2025-11-11T17:20:14.566487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518879450659189:2085] [node 6] ICS10 traffic start
2025-11-11T17:20:14.566523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:14.573898Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518879450659172:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:14.589915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518875155691808:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.609043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:14.609104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518875155691808:2048] [node 2] ICP22 created new session: [1:7571518879450659193:2048]
2025-11-11T17:20:14.617625Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659174:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.617665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518879450659174:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.617663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518879450659193:2048] [node 2] ICS09 handshake done sender: [1:7571518879450659172:2111] self: [1:7571518879450659171:0] peer: [2:7571518880518501270:0] socket: 438 qp: -1
2025-11-11T17:20:14.631514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518879450659192:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33784
2025-11-11T17:20:14.637778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518879450659193:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:14.645768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518879450659174:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:14.646618Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 2] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:14.647007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518875155691868:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.647108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:14.647151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518875155691868:2084] [node 5] ICP22 created new session: [1:7571518879450659197:2084]
2025-11-11T17:20:14.647187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518879450659197:2084] [node 5] ICS09 handshake done sender: [1:7571518879450659174:2112] self: [1:7571518879450659173:0] peer: [5:7571518881314264592:0] socket: 439 qp: -1
2025-11-11T17:20:14.647221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518879450659197:2084] [node 5] ICS10 traffic start
2025-11-11T17:20:14.647272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:14.650734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518879450659196:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33784
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.618537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518878003255065:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518882298222439:2116] held: no
2025-11-11T17:20:14.578332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518877019297210:2049] [node 2] ICP22 created new session: [5:7571518881314264588:2049]
2025-11-11T17:20:14.621836Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518880786248720:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.581712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518875668012114:2049] [node 2] ICP22 created new session: [3:7571518879962979499:2049]
2025-11-11T17:20:14.578373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518881314264588:2049] [node 2] ICS09 handshake done sender: [5:7571518881314264572:2111] self: [5:7571518881314264581:0] peer: [2:7571518880518501243:0] socket: 424 qp: -1
2025-11-11T17:20:14.578403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518881314264588:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:14.578439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:14.578717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518877019297210:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:14.685677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.697665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659196:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.618713Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.585531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518881314264562:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.621879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518880786248702:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.621925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:14.621974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518880786248702:2084] [node 6] ICP22 created new session: [4:7571518880786248721:2084]
2025-11-11T17:20:14.622018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518880786248721:2084] [node 6] ICS09 handshake done sender: [4:7571518880786248709:2112] self: [4:7571518880786248708:0] peer: [6:7571518882298222444:0] socket: 432 qp: -1
2025-11-11T17:20:14.622045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518880786248721:2084] [node 6] ICS10 traffic start
2025-11-11T17:20:14.622074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:14.622944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518876491281344:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.637993Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:14.643160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518876491281344:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518880786248720:2117] held: no
2025-11-11T17:20:14.643476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248725:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.643603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248726:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.643702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248727:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.657649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518880786248719:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.666965Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518880786248725:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.667076Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518880786248726:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.667129Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518880786248720:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:14.685774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518876491281344:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.685815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:14.685871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518876491281344:2050] [node 3] ICP22 created new session: [4:7571518880786248729:2050]
2025-11-11T17:20:14.693827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518876491281343:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.693921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518876491281342:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.713731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518880786248729:2050] [node 3] ICS09 handshake done sender: [4:7571518880786248720:2117] self: [4:7571518880786248724:0] peer: [3:7571518879962979486:0] socket: 459 qp: -1
2025-11-11T17:20:14.713790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518880786248729:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:14.713831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:14.589712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518879962979499:2049] [node 2] ICS09 handshake done sender: [3:7571518879962979490:2117] self: [3:7571518879962979492:0] peer: [2:7571518880518501245:0] socket: 426 qp: -1
2025-11-11T17:20:14.619850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222448:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.620464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222451:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.641733Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518882298222439:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:14.678154Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:14.685156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518878003255066:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518882298222426:2109] held: no
2025-11-11T17:20:14.687758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222452:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.688119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518878003255065:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.688192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:14.688232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518878003255065:2049] [node 2] ICP22 created new session: [6:7571518882298222453:2049]
2025-11-11T17:20:14.688270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518882298222453:2049] [node 2] ICS09 handshake done sender: [6:7571518882298222439:2116] self: [6:7571518882298222446:0] peer: [2:7571518880518501238:0] socket: 425 qp: -1
2025-11-11T17:20:14.688298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518882298222453:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:14.688322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:14.688782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518882298222377:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518882298222441:2118] held: no
2025-11-11T17:20:14.689294Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518882298222441:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:14.698053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518882298222377:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.698129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:14.698175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518882298222377:2062] [node 4] ICP22 created new session: [6:7571518882298222457:2062]
2025-11-11T17:20:14.698217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518882298222457:2062] [node 4] ICS09 handshake done sender: [6:7571518882298222441:2118] self: [6:7571518882298222444:0] peer: [4:7571518880786248708:0] socket: 437 qp: -1
2025-11-11T17:20:14.698249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518882298222457:2062] [node 4] ICS10 traffic start
2025-11-11T17:20:14.698278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:14.698639Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518882298222440:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:14.698956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518882298222384:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.699008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:14.699037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518882298222384:2066] [node 5] ICP22 created new session: [6:7571518882298222459:2066]
2025-11-11T17:20:14.699072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518882298222459:2066] [node 5] ICS09 handshake done sender: [6:7571518882298222440:2117] self: [6:7571518882298222445:0] peer: [5:7571518881314264573:0] socket: 435 qp: -1
2025-11-11T17:20:14.699094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518882298222459:2066] [node 5] ICS10 traffic start
2025-11-11T17:20:14.699136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:14.710278Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659181:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.733873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518876491281344:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(28192795ns) {HandleHandshakeStatus:329(28191228ns) {DropIncomingHandshake:441(614ns) DropOutgoingHandshake:456(163ns) SwitchToState:178(830ns) ProcessPendingSessionEvents:495(595ns)}}
2025-11-11T17:20:14.585621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:14.737835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518882298222450:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28167
2025-11-11T17:20:14.589749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518879962979499:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.714014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518876223533876:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(104102284ns) {HandleHandshakeStatus:329(104099075ns) {DropIncomingHandshake:441(714ns) DropOutgoingHandshake:456(5790ns) SwitchToState:178(523ns) ProcessPendingSessionEvents:495(76091369ns) {ForwardSessionEventToSession:612(40060476ns) ForwardSessionEventToSession:612(7576851ns) ForwardSessionEventToSession:612(3935ns) ForwardSessionEventToSession:612(3674ns) ForwardSessionEventToSession:612(12441917ns) ForwardSessionEventToSession:612(3208ns) ForwardSessionEventToSession:612(9422855ns) ForwardSessionEventToSession:612(5147ns) ForwardSessionEventToSession:612(6537712ns) ForwardSessionEventToSession:612(11956ns)}}}
2025-11-11T17:20:14.710328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518879450659181:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.780165Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518880786248714:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.585667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518881314264562:2063] [node 6] ICP22 created new session: [5:7571518881314264590:2063]
2025-11-11T17:20:14.758130Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518882298222442:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:14.770010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518878003255066:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(43846883ns) {HandleHandshakeStatus:395(34212575ns) {LogHandshakeFail:472(14543ns) {UpdateErrorStateLog:544(4276ns)} DropOutgoingHandshake:456(2956ns) ProcessPendingSessionEvents:495(28460430ns) {DropSessionEvent:512(10606ns) DropSessionEvent:512(5602ns) DropSessionEvent:512(4299ns) DropSessionEvent:512(12462362ns) DropSessionEvent:512(4770ns) DropSessionEvent:512(5076ns) DropSessionEvent:512(6225ns) DropSessionEvent:512(5966ns) DropSessionEvent:512(5760ns) DropSessionEvent:512(15941481ns)}}}
2025-11-11T17:20:14.774422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518878003255064:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.781793Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518882298222426:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.593850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:14.789804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:14.780205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518880786248714:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.729637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518880518501249:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.768354Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518879450659180:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.612707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518879962979501:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19985
2025-11-11T17:20:14.590176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518877019297209:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.590855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264591:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.613772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518881314264590:2063] [node 6] ICS09 handshake done sender: [5:7571518881314264574:2112] self: [5:7571518881314264573:0] peer: [6:7571518882298222445:0] socket: 428 qp: -1
2025-11-11T17:20:14.613817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518881314264590:2063] [node 6] ICS10 traffic start
2025-11-11T17:20:14.625786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:14.658066Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518881314264591:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.658257Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:14.687156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518881314264562:2063] [node 6] ICP35 event processing took too much time PendingConnection:278(101411617ns) {HandleHandshakeStatus:329(101408554ns) {DropIncomingHandshake:441(304ns) DropOutgoingHandshake:456(450ns) SwitchToState:178(467ns) ProcessPendingSessionEvents:495(9077933ns) {ForwardSessionEventToSession:612(9026345ns) ForwardSessionEventToSession:612(5029ns) ForwardSessionEventToSession:612(6533ns) ForwardSessionEventToSession:612(5315ns) ForwardSessionEventToSession:612(3277ns) ForwardSessionEventToSession:612(3360ns) ForwardSessionEventToSession:612(5376ns) ForwardSessionEventToSession:612(2491ns) ForwardSessionEventToSession:612(4793ns) ForwardSessionEventToSession:612(2807ns)}}}
2025-11-11T17:20:14.687329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518877019297209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518881314264585:2116] held: no
2025-11-11T17:20:14.689859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264597:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.742428Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518881314264585:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:14.742709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264598:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.742760Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:20:14.743139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518877019297209:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.743208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:14.743249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518877019297209:2048] [node 1] ICP22 created new session: [5:7571518881314264600:2048]
2025-11-11T17:20:14.743305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518881314264600:2048] [node 1] ICS09 handshake done sender: [5:7571518881314264585:2116] self: [5:7571518881314264592:0] peer: [1:7571518879450659173:0] socket: 441 qp: -1
2025-11-11T17:20:14.743339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518881314264600:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:14.743368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:14.749680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518877019297211:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.765937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264596:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.769908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518877019297211:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(22593032ns) {IncomingHandshake:261(22590630ns) {DropIncomingHandshake:441(886ns) PrepareNewSessionHandshake:476(6791ns) {SwitchToState:178(4046ns)} GenerateSessionVirtualId:988(1219ns) IssueIncomingHandshakeReply:193(66496ns) {SwitchToState:178(585ns)}}}
2025-11-11T17:20:14.779401Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518881314264596:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.779522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518877019297228:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.781378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264603:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.803369Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518881314264596:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:14.798035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518878003255064:2048] [node 1] ICP22 created new session: [6:7571518882298222461:2048]
2025-11-11T17:20:14.804226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518878003255066:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.804305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:14.804371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518878003255066:2050] [node 3] ICP22 created new session: [6:7571518882298222462:2050]
2025-11-11T17:20:14.804422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518882298222462:2050] [node 3] ICS09 handshake done sender: [6:7571518882298222426:2109] self: [6:7571518882298222443:0] peer: [3:7571518875668012170:0] socket: 415 qp: -1
2025-11-11T17:20:14.804456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518882298222462:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:14.804496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:14.805869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518882298222461:2048] [node 1] ICS09 handshake done sender: [6:7571518882298222442:2119] self: [6:7571518882298222447:0] peer: [1:7571518879450659175:0] socket: 442 qp: -1
2025-11-11T17:20:14.805918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518882298222461:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:14.805985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:14.806439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518878003255064:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(37642512ns) {HandleHandshakeStatus:329(37639798ns) {DropIncomingHandshake:441(2113ns) DropOutgoingHandshake:456(6176ns) SwitchToState:178(740ns) ProcessPendingSessionEvents:495(97919ns) {ForwardSessionEventToSession:612(54322ns) ForwardSessionEventToSession:612(3875ns) ForwardSessionEventToSession:612(3092ns) ForwardSessionEventToSession:612(6567ns) ForwardSessionEventToSession:612(5388ns) ForwardSessionEventToSession:612(2461ns) ForwardSessionEventToSession:612(6253ns) ForwardSessionEventToSession:612(3119ns) ForwardSessionEventToSession:612(5115ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.819238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518877019297228:2061] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.768391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518879450659180:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.781845Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518880786248714:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:14.729683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518880518501249:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.617893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518875668012114:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(52401852ns) {HandleHandshakeStatus:329(52398707ns) {DropIncomingHandshake:441(1904ns) DropOutgoingHandshake:456(426ns) SwitchToState:178(603ns) ProcessPendingSessionEvents:495(1208ns)}}
2025-11-11T17:20:14.802225Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:14.779126Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518879450659181:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:14.797738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518880518501249:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:14.831269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518880518501240:2085] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.831394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:14.831450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518880518501240:2085] [node 4] ICP22 created new session: [2:7571518880518501276:2085]
2025-11-11T17:20:14.847719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518880518501276:2085] [node 4] ICS09 handshake done sender: [2:7571518880518501249:2116] self: [2:7571518880518501248:0] peer: [4:7571518880786248730:0] socket: 433 qp: -1
2025-11-11T17:20:14.838094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:14.623358Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518879962979487:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.816615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518876491281342:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518880786248725:2118] held: no
2025-11-11T17:20:14.816718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518880786248701:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.816760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:14.816795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518880786248701:2082] [node 5] ICP22 created new session: [4:7571518880786248736:2082]
2025-11-11T17:20:14.816836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518880786248736:2082] [node 5] ICS09 handshake done sender: [4:7571518880786248714:2114] self: [4:7571518880786248713:0] peer: [5:7571518881314264602:0] socket: 417 qp: -1
2025-11-11T17:20:14.816865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518880786248736:2082] [node 5] ICS10 traffic start
2025-11-11T17:20:14.816897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:14.822985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248735:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.824181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518880786248738:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11283
2025-11-11T17:20:14.830352Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518880786248719:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:14.832283Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:14.833373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518876491281343:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.833476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:14.833525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518876491281343:2049] [node 2] ICP22 created new session: [4:7571518880786248739:2049]
2025-11-11T17:20:14.833572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518880786248739:2049] [node 2] ICS09 handshake done sender: [4:7571518880786248719:2116] self: [4:7571518880786248730:0] peer: [2:7571518880518501248:0] socket: 456 qp: -1
2025-11-11T17:20:14.833627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518880786248739:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:14.833690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:14.833929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518876491281343:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:14.834182Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518880786248725:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:14.834455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518876491281342:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.834537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:14.834577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518876491281342:2048] [node 1] ICP22 created new session: [4:7571518880786248741:2048]
2025-11-11T17:20:14.834612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518880786248741:2048] [node 1] ICS09 handshake done sender: [4:7571518880786248725:2118] self: [4:7571518880786248731:0] peer: [1:7571518879450659179:0] socket: 465 qp: -1
2025-11-11T17:20:14.834636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518880786248741:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:14.834666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:14.784259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518875155691808:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(151939365ns) {HandleHandshakeStatus:329(151932278ns) {DropIncomingHandshake:441(411ns) DropOutgoingHandshake:456(381ns) SwitchToState:178(397ns) ProcessPendingSessionEvents:495(13749566ns) {ForwardSessionEventToSession:612(13692698ns) ForwardSessionEventToSession:612(3395ns) ForwardSessionEventToSession:612(2915ns) ForwardSessionEventToSession:612(5510ns) ForwardSessionEventToSession:612(2918ns) ForwardSessionEventToSession:612(11830ns) ForwardSessionEventToSession:612(3116ns) ForwardSessionEventToSession:612(2575ns) ForwardSessionEventToSession:612(8721ns) ForwardSessionEventToSession:612(2222ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:14.623379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518879962979487:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.624192Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518879962979487:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:14.624407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518879962979479:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.624453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:14.624499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518879962979479:2075] [node 4] ICP22 created new session: [3:7571518879962979503:2075]
2025-11-11T17:20:14.624534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518879962979503:2075] [node 4] ICS09 handshake done sender: [3:7571518879962979487:2114] self: [3:7571518879962979486:0] peer: [4:7571518880786248724:0] socket: 458 qp: -1
2025-11-11T17:20:14.624560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518879962979503:2075] [node 4] ICS10 traffic start
2025-11-11T17:20:14.624587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:14.667493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518879962979505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19985
2025-11-11T17:20:14.679016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518879962979505:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.679164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518875668012113:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:14.679478Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518879962979501:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.737930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518875668012137:2071] [node 5] ICP35 event processing took too much time PendingConnection:278(30422862ns) {IncomingHandshake:261(18330909ns) {DropIncomingHandshake:441(1627ns) PrepareNewSessionHandshake:476(4165ns) {SwitchToState:178(2331ns)} GenerateSessionVirtualId:988(703ns) IssueIncomingHandshakeReply:193(6409ns)}}
2025-11-11T17:20:14.770057Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518879962979488:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:14.770090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518879962979488:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:14.778385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518879962979508:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19985
2025-11-11T17:20:14.780018Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518879962979505:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:14.781622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518875668012113:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.786877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:14.786922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518875668012113:2048] [node 1] ICP22 created new session: [3:7571518879962979509:2048]
2025-11-11T17:20:14.802000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518879962979509:2048] [node 1] ICS09 handshake done sender: [3:7571518879962979505:2120] self: [3:7571518879962979506:0] peer: [1:7571518879450659178:0] socket: 410 qp: -1
2025-11-11T17:20:14.814927Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518879962979488:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:14.815914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518875668012137:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.815981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:14.816033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518875668012137:2071] [node 5] ICP22 created new session: [3:7571518879962979511:2071]
2025-11-11T17:20:14.816077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518879962979511:2071] [node 5] ICS09 handshake done sender: [3:7571518879962979488:2115] self: [3:7571518879962979485:0] peer: [5:7571518881314264599:0] socket: 408 qp: -1
2025-11-11T17:20:14.816112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518879962979511:2071] [node 5] ICS10 traffic start
2025-11-11T17:20:14.816148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:14.818132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518879962979509:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:14.825806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:14.858021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518875668012137:2071] [node 5] ICP35 event processing took too much time PendingConnection:278(41868774ns) {HandleHandshakeStatus:329(41864964ns) {DropIncomingHandshake:441(18207ns) DropOutgoingHandshake:456(528ns) SwitchToState:178(512ns) ProcessPendingSessionEvents:495(11963769ns) {ForwardSessionEventToSession:612(11907288ns) ForwardSessionEventToSession:612(7219ns) ForwardSessionEventToSession:612(3223ns) ForwardSessionEventToSession:612(8706ns) ForwardSessionEventToSession:612(2175ns) ForwardSessionEventToSession:612(2754ns) ForwardSessionEventToSession:612(8060ns) ForwardSessionEventToSession:612(2676ns) ForwardSessionEventToSession:612(5029ns)}}}
2025-11-11T17:20:14.866789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518875668012113:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(84980088ns) {HandleHandshakeStatus:329(84972556ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(4506ns) SwitchToState:178(297ns) ProcessPendingSessionEvents:495(8022410ns) {ForwardSessionEventToSession:612(7966383ns) ForwardSessionEventToSession:612(11602ns) ForwardSessionEventToSession:612(3139ns) ForwardSessionEventToSession:612(3308ns) ForwardSessionEventToSession:612(5183ns) ForwardSessionEventToSession:612(5810ns) ForwardSessionEventToSession:612(3115ns) ForwardSessionEventToSession:612(5483ns) ForwardSessionEventToSession:612(2517ns) ForwardSessionEventToSession:612(4616ns)}}}
2025-11-11T17:20:14.858209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518880518501276:2085] [node 4] ICS10 traffic start
2025-11-11T17:20:14.858270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:14.872341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518880518501240:2085] [node 4] ICP35 event processing took too much time PendingConnection:278(40826266ns) {HandleHandshakeStatus:329(40823976ns) {DropIncomingHandshake:441(15178ns) DropOutgoingHandshake:456(553ns) SwitchToState:178(1055ns) ProcessPendingSessionEvents:495(112206ns) {ForwardSessionEventToSession:612(57701ns) ForwardSessionEventToSession:612(9339ns) ForwardSessionEventToSession:612(2827ns) ForwardSessionEventToSession:612(3127ns) ForwardSessionEventToSession:612(5146ns) ForwardSessionEventToSession:612(3010ns) ForwardSessionEventToSession:612(6372ns) ForwardSessionEventToSession:612(2949ns) ForwardSessionEventToSession:612(4824ns) ForwardSessionEventToSession:612(5470ns)}}}
2025-11-11T17:20:14.786999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518875155691808:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:14.790099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518875155691809:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.790199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:14.790242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518875155691809:2049] [node 3] ICP22 created new session: [1:7571518879450659204:2049]
2025-11-11T17:20:14.790289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518879450659204:2049] [node 3] ICS09 handshake done sender: [1:7571518879450659181:2116] self: [1:7571518879450659178:0] peer: [3:7571518879962979506:0] socket: 467 qp: -1
2025-11-11T17:20:14.790335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518879450659204:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:14.790367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:14.791465Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518879450659180:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:14.791802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518875155691862:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:14.791876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:14.791921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518875155691862:2086] [node 4] ICP22 created new session: [1:7571518879450659206:2086]
2025-11-11T17:20:14.791970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518879450659206:2086] [node 4] ICS09 handshake done sender: [1:7571518879450659180:2115] self: [1:7571518879450659179:0] peer: [4:7571518880786248731:0] socket: 462 qp: -1
2025-11-11T17:20:14.791998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518879450659206:2086] [node 4] ICS10 traffic start
2025-11-11T17:20:14.792028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:14.805674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518879450659201:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33784
2025-11-11T17:20:14.837842Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:14.849861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518875155691863:2085] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:14.850282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518877019297228:2061] [node 4] ICP22 created new session: [5:7571518881314264604:2061]
2025-11-11T17:20:14.855862Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:14.856806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518877019297211:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518881314264591:2118] held: no
2025-11-11T17:20:14.861733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518881314264604:2061] [node 4] ICS09 handshake done sender: [5:7571518881314264596:2119] self: [5:7571518881314264602:0] peer: [4:7571518880786248713:0] socket: 434 qp: -1
2025-11-11T17:20:14.869813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518881314264604:2061] [node 4] ICS10 traffic start
2025-11-11T17:20:14.871156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:14.881941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518881314264606:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16452
2025-11-11T17:20:14.886284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518877019297228:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(66816372ns) {HandleHandshakeStatus:329(66813502ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(5025ns) SwitchToState:178(650ns) ProcessPendingSessionEvents:495(121901ns) {ForwardSessionEventToSession:612(73428ns) ForwardSessionEventToSession:612(4718ns) ForwardSessionEventToSession:612(3379ns) ForwardSessionEventToSession:612(6058ns) ForwardSessionEventToSession:612(3485ns) ForwardSessionEventToSession:612(5699ns) ForwardSessionEventToSession:612(5369ns) ForwardSessionEventToSession:612(3038ns) ForwardSessionEventToSession:612(4786ns) ForwardSessionEventToSession:612(3418ns)}}}
2025-11-11T17:20:14.887965Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518881314264591:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:14.890003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518877019297211:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:14.890092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:14.890141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518877019297211:2050] [node 3] ICP22 created new session: [5:7571518881314264607:2050]
2025-11-11T17:20:14.890191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518881314264607:2050] [node 3] ICS09 handshake done sender: [5:7571518881314264591:2118] self: [5:7571518881314264599:0] peer: [3:7571518879962979485:0] socket: 429 qp: -1
2025-11-11T17:20:14.890226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518881314264607:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:14.890275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:14.956489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:14.956708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518880518501273:2048] [node 1] ICS01 socket: 444 reason# EndOfStream
2025-11-11T17:20:14.956753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518876223533876:2048] [node 1] ICP30 unregister session Session# [2:7571518880518501273:2048] VirtualId# [2:7571518880518501270:0]
2025-11-11T17:20:14.956791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518880518501273:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:14.956866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:14.956905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:14.958682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:14.958896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518879962979509:2048] [node 1] ICS07 socket disconnect 410 reason# EndOfStream
2025-11-11T17:20:14.958922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518879962979509:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:14.958984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:14.959021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518879962979509:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:14.961573Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:20:14.961717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518875668012113:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:20:14.961753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518879962979509:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:14.962765Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Couldn't connect to any resolved address
2025-11-11T17:20:14.963022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518875668012113:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:14.963070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518879962979509:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:14.963102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518875668012113:2048] [node 1] ICP30 unregister session Session# [3:7571518879962979509:2048] VirtualId# [3:7571518879962979506:0]
2025-11-11T17:20:14.963145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:14.963202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518875668012113:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:14.963219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:14.970646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:14.970128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:14.970842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518882298222461:2048] [node 1] ICS01 socket: 442 reason# EndOfStream
2025-11-11T17:20:14.970875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518878003255064:2048] [node 1] ICP30 unregister session Session# [6:7571518882298222461:2048] VirtualId# [6:7571518882298222447:0]
2025-11-11T17:20:14.970900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518882298222461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:14.970960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:14.970997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:14.972080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518875668012113:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:14.970328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518881314264600:2048] [node 1] ICS01 socket: 441 reason# EndOfStream
2025-11-11T17:20:14.970371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518877019297209:2048] [node 1] ICP30 unregister session Session# [5:7571518881314264600:2048] VirtualId# [5:7571518881314264592:0]
2025-11-11T17:20:14.970402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518881314264600:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:14.970524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:14.970566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:14.977720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:14.977959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518880786248741:2048] [node 1] ICS01 socket: 465 reason# EndOfStream
2025-11-11T17:20:14.978000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518876491281342:2048] [node 1] ICP30 unregister session Session# [4:7571518880786248741:2048] VirtualId# [4:7571518880786248731:0]
2025-11-11T17:20:14.978040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518880786248741:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:14.978166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:14.978215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:15.039621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:15.039923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518879962979499:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-11-11T17:20:15.039980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518875668012114:2049] [node 2] ICP30 unregister session Session# [3:7571518879962979499:2049] VirtualId# [3:7571518879962979492:0]
2025-11-11T17:20:15.040034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518879962979499:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.040181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:15.040235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:15.043022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:15.043271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518881314264588:2049] [node 2] ICS01 socket: 424 reason# EndOfStream
2025-11-11T17:20:15.043320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518877019297210:2049] [node 2] ICP30 unregister session Session# [5:7571518881314264588:2049] VirtualId# [5:7571518881314264581:0]
2025-11-11T17:20:15.043367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518881314264588:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.043477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:15.043539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:15.045771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:15.045998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518882298222453:2049] [node 2] ICS01 socket: 425 reason# EndOfStream
2025-11-11T17:20:15.046039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518878003255065:2049] [node 2] ICP30 unregister session Session# [6:7571518882298222453:2049] VirtualId# [6:7571518882298222446:0]
2025-11-11T17:20:15.046071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518882298222453:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.046155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:15.046200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:15.049835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:15.050214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518880786248739:2049] [node 2] ICS01 socket: 456 reason# EndOfStream
2025-11-11T17:20:15.050270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518876491281343:2049] [node 2] ICP30 unregister session Session# [4:7571518880786248739:2049] VirtualId# [4:7571518880786248730:0]
2025-11-11T17:20:15.050327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518880786248739:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.050495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:15.050569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:15.138385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:15.138643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518882298222462:2050] [node 3] ICS01 socket: 415 reason# EndOfStream
2025-11-11T17:20:15.138698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518878003255066:2050] [node 3] ICP30 unregister session Session# [6:7571518882298222462:2050] VirtualId# [6:7571518882298222443:0]
2025-11-11T17:20:15.141728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:15.138745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518882298222462:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.138842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:15.138891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:15.142040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518881314264607:2050] [node 3] ICS01 socket: 429 reason# EndOfStream
2025-11-11T17:20:15.142128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518877019297211:2050] [node 3] ICP30 unregister session Session# [5:7571518881314264607:2050] VirtualId# [5:7571518881314264599:0]
2025-11-11T17:20:15.142180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518881314264607:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.142348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:15.142438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:15.141759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:15.142100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518880786248729:2050] [node 3] ICS01 socket: 459 reason# EndOfStream
2025-11-11T17:20:15.142163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518876491281344:2050] [node 3] ICP30 unregister session Session# [4:7571518880786248729:2050] VirtualId# [4:7571518880786248724:0]
2025-11-11T17:20:15.142224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518880786248729:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.142418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:15.142496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:15.221793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:15.222025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518881314264604:2061] [node 4] ICS01 socket: 434 reason# EndOfStream
2025-11-11T17:20:15.222074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518877019297228:2061] [node 4] ICP30 unregister session Session# [5:7571518881314264604:2061] VirtualId# [5:7571518881314264602:0]
2025-11-11T17:20:15.222112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518881314264604:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.222221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:15.222272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:15.222372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:15.222603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518882298222457:2062] [node 4] ICS01 socket: 437 reason# EndOfStream
2025-11-11T17:20:15.222634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518882298222377:2062] [node 4] ICP30 unregister session Session# [6:7571518882298222457:2062] VirtualId# [6:7571518882298222444:0]
2025-11-11T17:20:15.222664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518882298222457:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.222724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:15.222763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:15.285863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:15.286253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518882298222459:2066] [node 5] ICS01 socket: 435 reason# EndOfStream
2025-11-11T17:20:15.286336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518882298222384:2066] [node 5] ICP30 unregister session Session# [6:7571518882298222459:2066] VirtualId# [6:7571518882298222445:0]
2025-11-11T17:20:15.286400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518882298222459:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:15.286574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:15.286668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:15.402432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518885381755938:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:15.402476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518885381755939:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:15.430209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518885736546793:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:15.430270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518885736546794:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:15.446330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518884800116172:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:15.446374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518884800116173:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:15.485829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518885986856961:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:15.485878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518885986856962:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:15.485908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518885986856963:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:15.528261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518886914570613:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:15.528309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518886914570614:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:15.528348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518886914570615:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:15.562945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518885934463008:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:15.562988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518885934463009:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:15.563018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518885934463010:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:15.976833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518885381755999:2107] [node 5] ICP01 ready to work
2025-11-11T17:20:15.982179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518885736546854:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:15.978666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518885986857019:2103] [node 6] ICP01 ready to work
2025-11-11T17:20:15.989170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518884800116224:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1641
2025-11-11T17:20:16.048121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518890031514154:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:15.993764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518885986857020:2107] [node 5] ICP01 ready to work
2025-11-11T17:20:16.042168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518885381755938:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(64166416ns) {Configure:104(64159251ns) {ProcessConfigured:135(90079ns) {StartInitialHandshake:157(87885ns) {DropHandshakes:469(3294ns) {DropIncomingHandshake:441(710ns) DropOutgoingHandshake:456(606ns)} GenerateSessionVirtualId:988(1131ns) PrepareNewSessionHandshake:476(2105ns) {SwitchToState:178(704ns)}}}}}
2025-11-11T17:20:16.049760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518889095083532:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:16.087566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518889095083532:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21627788ns) {Configure:104(21622741ns) {ProcessConfigured:135(66668ns) {StartInitialHandshake:157(61964ns) {DropHandshakes:469(1336ns) {DropIncomingHandshake:441(316ns) DropOutgoingHandshake:456(218ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:476(1097ns) {SwitchToState:178(373ns)}}}}}
2025-11-11T17:20:16.042629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518889676723300:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:16.082041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518885381755939:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(101406103ns) {Configure:104(101400974ns) {ProcessConfigured:135(16119142ns) {StartInitialHandshake:157(16116619ns) {DropHandshakes:469(2240ns) {DropIncomingHandshake:441(434ns) DropOutgoingHandshake:456(506ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:476(3113ns) {SwitchToState:178(1483ns)}}}}}
2025-11-11T17:20:16.049771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518890031514153:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:16.058980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518891209537955:2079] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.106052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518885934463010:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(127559479ns) {Configure:104(127551419ns) {ProcessConfigured:135(9621032ns) {StartInitialHandshake:157(9618133ns) {DropHandshakes:469(1895ns) {DropIncomingHandshake:441(457ns) DropOutgoingHandshake:456(360ns)} GenerateSessionVirtualId:988(774ns) PrepareNewSessionHandshake:476(3894ns) {SwitchToState:178(1915ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.094608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518890031514154:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35292251ns) {Configure:104(35284927ns) {ProcessConfigured:135(51095ns) {StartInitialHandshake:157(48872ns) {DropHandshakes:469(926ns) {DropIncomingHandshake:441(253ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:476(1189ns) {SwitchToState:178(495ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.106348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518886914570613:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(100766526ns) {Configure:104(100758712ns) {ProcessConfigured:135(78564ns) {StartInitialHandshake:157(76299ns) {DropHandshakes:469(1968ns) {DropIncomingHandshake:441(402ns) DropOutgoingHandshake:456(255ns)} GenerateSessionVirtualId:988(1301ns) PrepareNewSessionHandshake:476(1901ns) {SwitchToState:178(756ns)}}}}}
2025-11-11T17:20:16.121130Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518884800116224:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.095199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518885736546854:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(38157033ns) {Configure:104(38151844ns) {ProcessConfigured:135(56717ns) {StartInitialHandshake:157(53609ns) {DropHandshakes:469(4758ns) {DropIncomingHandshake:441(382ns) DropOutgoingHandshake:456(366ns)} GenerateSessionVirtualId:988(782ns) PrepareNewSessionHandshake:476(1596ns) {SwitchToState:178(746ns)}}}}}
2025-11-11T17:20:16.114870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518886914570614:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(128559525ns) {Configure:104(128553595ns) {ProcessConfigured:135(4740707ns) {StartInitialHandshake:157(4737740ns) {DropHandshakes:469(2155ns) {DropIncomingHandshake:441(386ns) DropOutgoingHandshake:456(362ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:476(3623ns) {SwitchToState:178(1555ns)}}}}}
2025-11-11T17:20:16.269740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518889676723309:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:16.106247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518890031514155:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16239
2025-11-11T17:20:16.127394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518889095083539:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.189008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430331:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.342175Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723306:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.342206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518889676723306:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.343481Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518889676723306:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:16.125458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518891209537971:2100] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.249300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518889095083547:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1641
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.189130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430332:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.106366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518890031514156:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16239
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.369654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824328:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
2025-11-11T17:20:16.189382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430333:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.305651Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518890229430332:2074] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.311390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518890229430374:2072] [node 5] ICP01 ready to work
2025-11-11T17:20:16.360524Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518890229430331:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.370366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518885934463010:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.372078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518890229430374:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.380498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518890229430376:2070] [node 4] ICP01 ready to work
2025-11-11T17:20:16.198445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209537967:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.355305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518885381755999:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(274686972ns) {Configure:104(274677749ns) {ProcessConfigured:135(129748670ns) {StartInitialHandshake:157(129744513ns) {DropHandshakes:469(3102ns) {DropIncomingHandshake:441(783ns) DropOutgoingHandshake:456(512ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:476(4670ns) {SwitchToState:178(2256ns)}}}}}
2025-11-11T17:20:16.253673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518889095083542:2109] [node 4] ICP01 ready to work
2025-11-11T17:20:16.293576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518889095083543:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1641
2025-11-11T17:20:16.341756Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518889095083543:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.341897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518884800116172:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.343288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518889095083550:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1641
2025-11-11T17:20:16.344075Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518889095083543:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:16.344266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518884800116172:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.344318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:16.344357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518884800116172:2048] [node 1] ICP22 created new session: [3:7571518889095083551:2048]
2025-11-11T17:20:16.344397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518889095083551:2048] [node 1] ICS09 handshake done sender: [3:7571518889095083543:2114] self: [3:7571518889095083549:0] peer: [1:7571518889676723304:0] socket: 431 qp: -1
2025-11-11T17:20:16.344430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518889095083551:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:16.344466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:16.402448Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518889095083534:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.402478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518889095083534:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.405304Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518889095083534:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:16.407022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518889095083547:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.407081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518889095083532:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.407127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.415094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518885986856961:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(339927391ns) {Configure:104(339922011ns) {ProcessConfigured:135(15877609ns) {StartInitialHandshake:157(15874305ns) {DropHandshakes:469(3190ns) {DropIncomingHandshake:441(1137ns) DropOutgoingHandshake:456(595ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:476(3941ns) {SwitchToState:178(1412ns)}}}}}
2025-11-11T17:20:16.421858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518885986857019:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(374764726ns) {Configure:104(369603164ns) {ProcessConfigured:135(12000945ns) {StartInitialHandshake:157(11996766ns) {DropHandshakes:469(3723ns) {DropIncomingHandshake:441(761ns) DropOutgoingHandshake:456(877ns)} GenerateSessionVirtualId:988(1268ns) PrepareNewSessionHandshake:476(6630ns) {SwitchToState:178(2605ns)}}}}}
2025-11-11T17:20:16.403764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518890229430333:2075] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.250461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209537976:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.355657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518889676723314:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3142
2025-11-11T17:20:16.394053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518890031514163:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16239
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.422041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518885736546793:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(395304814ns) {Configure:104(395293222ns) {ProcessConfigured:135(12031719ns) {StartInitialHandshake:157(12027720ns) {DropHandshakes:469(2083ns) {DropIncomingHandshake:441(381ns) DropOutgoingHandshake:456(521ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:476(4659ns) {SwitchToState:178(2177ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.421973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518889095083532:2108] [node 6] ICP22 created new session: [3:7571518889095083554:2108]
2025-11-11T17:20:16.406923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518885934463008:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.362224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518885381755939:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.415635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518891209537967:2097] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.437965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824334:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
2025-11-11T17:20:16.422072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518889095083554:2108] [node 6] ICS09 handshake done sender: [3:7571518889095083534:2111] self: [3:7571518889095083533:0] peer: [6:7571518890229430377:0] socket: 422 qp: -1
2025-11-11T17:20:16.442815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514156:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.438760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430383:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.362277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-11T17:20:16.421888Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518891209537959:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.450574Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518890281824328:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.442906Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514163:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.443115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518885736546793:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.450038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514155:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.451363Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514158:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.451386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518890031514158:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.463356Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518890031514158:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:16.490246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518890031514153:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(394712499ns) {Configure:104(394707295ns) {ProcessConfigured:135(12596865ns) {StartInitialHandshake:157(12594087ns) {DropHandshakes:469(2977ns) {DropIncomingHandshake:441(459ns) DropOutgoingHandshake:456(618ns)} GenerateSessionVirtualId:988(668ns) PrepareNewSessionHandshake:476(3150ns) {SwitchToState:178(985ns)}}}}}
2025-11-11T17:20:16.537835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518885736546794:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(459969112ns) {Configure:104(454680790ns) {ProcessConfigured:135(45654715ns) {StartInitialHandshake:157(45650315ns) {DropHandshakes:469(2665ns) {DropIncomingHandshake:441(437ns) DropOutgoingHandshake:456(769ns)} GenerateSessionVirtualId:988(975ns) PrepareNewSessionHandshake:476(20902874ns) {SwitchToState:178(5156ns)}}}}}
2025-11-11T17:20:16.461111Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518890229430331:2073] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:16.422120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518889095083554:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:16.362327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518885381755939:2049] [node 3] ICP22 created new session: [1:7571518889676723317:2049]
2025-11-11T17:20:16.421944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518891209537959:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.450725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518885986856962:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.440452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:16.518393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518889095083532:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(111038749ns) {HandleHandshakeStatus:329(111035389ns) {DropIncomingHandshake:441(471ns) DropOutgoingHandshake:456(677ns) SwitchToState:178(759ns) ProcessPendingSessionEvents:495(31613024ns) {ForwardSessionEventToSession:612(31535893ns) ForwardSessionEventToSession:612(24632ns) ForwardSessionEventToSession:612(6546ns) ForwardSessionEventToSession:612(5554ns) ForwardSessionEventToSession:612(2745ns) ForwardSessionEventToSession:612(2532ns) ForwardSessionEventToSession:612(5597ns) ForwardSessionEventToSession:612(3139ns) ForwardSessionEventToSession:612(2701ns) ForwardSessionEventToSession:612(5083ns)}}}
2025-11-11T17:20:16.518537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571518889095083532:2108] [node 6] ICP18 handshake (actor [3:7571518889095083547:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.568856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518890031514175:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16239
2025-11-11T17:20:16.574834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518885736546854:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.574922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:16.574972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518885736546854:2107] [node 4] ICP22 created new session: [2:7571518890031514177:2107]
2025-11-11T17:20:16.575015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518890031514177:2107] [node 4] ICS09 handshake done sender: [2:7571518890031514158:2112] self: [2:7571518890031514157:0] peer: [4:7571518890281824335:0] socket: 428 qp: -1
2025-11-11T17:20:16.575051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518890031514177:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:16.575093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:16.575702Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518890031514156:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:16.461347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518885934463010:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.452622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824336:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
2025-11-11T17:20:16.362376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518889676723317:2049] [node 3] ICS09 handshake done sender: [1:7571518889676723306:2112] self: [1:7571518889676723304:0] peer: [3:7571518889095083549:0] socket: 425 qp: -1
2025-11-11T17:20:16.442117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209537989:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.461402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:16.461451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518885934463010:2050] [node 3] ICP22 created new session: [6:7571518890229430386:2050]
2025-11-11T17:20:16.461499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518890229430386:2050] [node 3] ICS09 handshake done sender: [6:7571518890229430331:2073] self: [6:7571518890229430377:0] peer: [3:7571518889095083533:0] socket: 432 qp: -1
2025-11-11T17:20:16.461528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518890229430386:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:16.461559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:16.525710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518885934463010:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(64121802ns) {HandleHandshakeStatus:329(64120173ns) {DropIncomingHandshake:441(879ns) DropOutgoingHandshake:456(4975ns) SwitchToState:178(643ns) ProcessPendingSessionEvents:495(18673614ns) {ForwardSessionEventToSession:612(18614788ns) ForwardSessionEventToSession:612(9720ns) ForwardSessionEventToSession:612(2706ns) ForwardSessionEventToSession:612(5249ns) ForwardSessionEventToSession:612(2561ns) ForwardSessionEventToSession:612(4387ns) ForwardSessionEventToSession:612(5417ns) ForwardSessionEventToSession:612(3441ns) ForwardSessionEventToSession:612(7132ns) ForwardSessionEventToSession:612(7291ns)}}}
2025-11-11T17:20:16.532199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430385:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.554529Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518890229430332:2074] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:16.583192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518890229430376:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(170641817ns) {Configure:104(170633631ns) {ProcessConfigured:135(105811ns) {StartInitialHandshake:157(103084ns) {DropHandshakes:469(17565ns) {DropIncomingHandshake:441(600ns) DropOutgoingHandshake:456(443ns)} GenerateSessionVirtualId:988(752ns) PrepareNewSessionHandshake:476(1967ns) {SwitchToState:178(479ns)}}}}}
2025-11-11T17:20:16.583334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430389:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.598555Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19969) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.615294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518890229430374:2072] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.615937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518890229430333:2075] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.618620Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518889095083541:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.618688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518889095083541:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.622269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518889095083557:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1641
2025-11-11T17:20:16.443674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209537987:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.577320Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514160:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.577353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518890031514160:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.589906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518885736546793:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.613931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:16.613996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518885736546793:2048] [node 1] ICP22 created new session: [2:7571518890031514181:2048]
2025-11-11T17:20:16.617670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518890031514176:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16239
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.632348Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514176:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.634207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518890031514182:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16239
2025-11-11T17:20:16.635917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518890031514181:2048] [node 1] ICS09 handshake done sender: [2:7571518890031514156:2111] self: [2:7571518890031514168:0] peer: [1:7571518889676723298:0] socket: 420 qp: -1
2025-11-11T17:20:16.637671Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518890031514160:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.639512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518889095083560:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1641
2025-11-11T17:20:16.617120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:16.644715Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518889095083541:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:16.641876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518890031514181:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.362410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518889676723317:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.497931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518885986856963:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(414284158ns) {Configure:104(414277803ns) {ProcessConfigured:135(19838651ns) {StartInitialHandshake:157(19835797ns) {DropHandshakes:469(1142ns) {DropIncomingHandshake:441(273ns) DropOutgoingHandshake:456(239ns)} GenerateSessionVirtualId:988(313ns) PrepareNewSessionHandshake:476(4117ns) {SwitchToState:178(1495ns)}}}}}
2025-11-11T17:20:16.648029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518889095083539:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.648125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:16.648173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518889095083539:2110] [node 5] ICP22 created new session: [3:7571518889095083561:2110]
2025-11-11T17:20:16.650776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518889095083560:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.654406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518889095083542:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(354694119ns) {Configure:104(354684384ns) {ProcessConfigured:135(22570039ns) {StartInitialHandshake:157(22566897ns) {DropHandshakes:469(2716ns) {DropIncomingHandshake:441(372ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(808ns) PrepareNewSessionHandshake:476(3697ns) {SwitchToState:178(1132ns)}}}}}
2025-11-11T17:20:16.653800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:16.522790Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518890281824328:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:16.523628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518885986856962:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.523704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:16.523760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518885986856962:2049] [node 2] ICP22 created new session: [4:7571518890281824339:2049]
2025-11-11T17:20:16.523811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518890281824339:2049] [node 2] ICS09 handshake done sender: [4:7571518890281824328:2112] self: [4:7571518890281824335:0] peer: [2:7571518890031514157:0] socket: 440 qp: -1
2025-11-11T17:20:16.523851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518890281824339:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:16.523892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:16.558683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518890281824334:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.558912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518885986856961:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.565838Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518890281824324:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.565890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518890281824324:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.620563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
2025-11-11T17:20:16.637837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518890281824324:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:16.638354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518885986857020:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.638424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:16.638475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518885986857020:2107] [node 5] ICP22 created new session: [4:7571518890281824345:2107]
2025-11-11T17:20:16.638539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518890281824345:2107] [node 5] ICS09 handshake done sender: [4:7571518890281824324:2111] self: [4:7571518890281824323:0] peer: [5:7571518891209537992:0] socket: 417 qp: -1
2025-11-11T17:20:16.638578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518890281824345:2107] [node 5] ICS10 traffic start
2025-11-11T17:20:16.638615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:16.647769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824348:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.453974Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518891209537959:2094] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:16.362446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:16.673923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518890031514154:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.674023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:16.674086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518890031514154:2109] [node 5] ICP22 created new session: [2:7571518890031514186:2109]
2025-11-11T17:20:16.674135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518890031514186:2109] [node 5] ICS09 handshake done sender: [2:7571518890031514160:2113] self: [2:7571518890031514159:0] peer: [5:7571518891209537999:0] socket: 429 qp: -1
2025-11-11T17:20:16.674170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518890031514186:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:16.674210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.619384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430390:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.678606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518889095083561:2110] [node 5] ICS09 handshake done sender: [3:7571518889095083541:2113] self: [3:7571518889095083540:0] peer: [5:7571518891209537998:0] socket: 453 qp: -1
2025-11-11T17:20:16.687123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518889095083561:2110] [node 5] ICS10 traffic start
2025-11-11T17:20:16.687177Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518889095083557:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.687185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:16.363422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518889676723315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3142
2025-11-11T17:20:16.539084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518891209537971:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(413248942ns) {Configure:104(413243463ns) {ProcessConfigured:135(17049034ns) {StartInitialHandshake:157(17045150ns) {DropHandshakes:469(5812620ns) {DropIncomingHandshake:441(1068ns) DropOutgoingHandshake:456(3383ns)} GenerateSessionVirtualId:988(2505ns) PrepareNewSessionHandshake:476(3745ns) {SwitchToState:178(2216ns)}}}}}
2025-11-11T17:20:16.668598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518890281824332:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.636003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518890229430374:2072] [node 5] ICP22 created new session: [6:7571518890229430394:2072]
2025-11-11T17:20:16.668645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518890281824332:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.678574Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518890281824334:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:16.686698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824350:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
2025-11-11T17:20:16.693064Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518890281824332:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:16.709758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518890281824350:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.706317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518884800116173:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.363589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518889676723316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3142
2025-11-11T17:20:16.539257Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518891209537987:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.729755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518885986856961:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.733699Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.641681Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518890229430390:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.700581Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514171:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.742303Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518890281824343:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.742355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518885986857019:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.742420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:16.742647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518885986856963:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.744381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518885986857019:2103] [node 6] ICP22 created new session: [4:7571518890281824352:2103]
2025-11-11T17:20:16.753848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.401972Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:16.405823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723314:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.405887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518885381755939:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:16.406413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518889676723319:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3142
2025-11-11T17:20:16.481907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723299:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.481943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518889676723299:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.489685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723302:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.489748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518889676723302:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.490586Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723316:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.755515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518890281824352:2103] [node 6] ICS09 handshake done sender: [4:7571518890281824332:2114] self: [4:7571518890281824331:0] peer: [6:7571518890229430401:0] socket: 463 qp: -1
2025-11-11T17:20:16.539612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518891209537971:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.645895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518885934463008:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.700622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518890031514171:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.743197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518889095083539:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(95005023ns) {HandleHandshakeStatus:329(94998171ns) {DropIncomingHandshake:441(18596ns) DropOutgoingHandshake:456(741ns) SwitchToState:178(1146ns) ProcessPendingSessionEvents:495(15945795ns) {ForwardSessionEventToSession:612(15891741ns) ForwardSessionEventToSession:612(6931ns) ForwardSessionEventToSession:612(9852ns) ForwardSessionEventToSession:612(5994ns) ForwardSessionEventToSession:612(5102ns) ForwardSessionEventToSession:612(3114ns) ForwardSessionEventToSession:612(2689ns) ForwardSessionEventToSession:612(5907ns) ForwardSessionEventToSession:612(2983ns) ForwardSessionEventToSession:612(2308ns)}}}
2025-11-11T17:20:16.490680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723319:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.566237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518891209537955:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.645977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:16.763609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518890281824352:2103] [node 6] ICS10 traffic start
2025-11-11T17:20:16.747887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518889095083567:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1641
2025-11-11T17:20:16.767195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824354:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
2025-11-11T17:20:16.773901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518885986856961:2048] [node 1] ICP22 created new session: [4:7571518890281824351:2048]
2025-11-11T17:20:16.774735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518890281824356:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32050
2025-11-11T17:20:16.775519Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518890281824350:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:16.777057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518885986856963:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.777131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:16.777201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518885986856963:2050] [node 3] ICP22 created new session: [4:7571518890281824357:2050]
2025-11-11T17:20:16.777248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518890281824357:2050] [node 3] ICS09 handshake done sender: [4:7571518890281824350:2120] self: [4:7571518890281824353:0] peer: [3:7571518889095083558:0] socket: 415 qp: -1
2025-11-11T17:20:16.777282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518890281824357:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:16.777317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:16.783262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:16.795736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518890281824351:2048] [node 1] ICS09 handshake done sender: [4:7571518890281824334:2115] self: [4:7571518890281824342:0] peer: [1:7571518889676723311:0] socket: 452 qp: -1
2025-11-11T17:20:16.801718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518890281824351:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:16.809751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:16.822135Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518890281824354:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.490959Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518889676723299:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:16.566368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:16.834044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518885986857020:2107] [node 5] ICP18 handshake (actor [4:7571518890281824354:2121]) is too old
2025-11-11T17:20:16.646029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518885934463008:2048] [node 1] ICP22 created new session: [6:7571518890229430397:2048]
2025-11-11T17:20:16.705542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518885736546793:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(126498991ns) {HandleHandshakeStatus:329(126496681ns) {DropIncomingHandshake:441(1442ns) DropOutgoingHandshake:456(16811ns) SwitchToState:178(1660ns) ProcessPendingSessionEvents:495(18207592ns) {ForwardSessionEventToSession:612(18140548ns) ForwardSessionEventToSession:612(6845ns) ForwardSessionEventToSession:612(7991ns) ForwardSessionEventToSession:612(6247ns) ForwardSessionEventToSession:612(3133ns) ForwardSessionEventToSession:612(16411ns) ForwardSessionEventToSession:612(2405ns) ForwardSessionEventToSession:612(2630ns) ForwardSessionEventToSession:612(6501ns) ForwardSessionEventToSession:612(2815ns)}}}
2025-11-11T17:20:16.646102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518890229430397:2048] [node 1] ICS09 handshake done sender: [6:7571518890229430333:2075] self: [6:7571518890229430384:0] peer: [1:7571518889676723301:0] socket: 434 qp: -1
2025-11-11T17:20:16.491256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518885381755938:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.757640Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518889095083562:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.566422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518891209537955:2079] [node 6] ICP22 created new session: [5:7571518891209537994:2079]
2025-11-11T17:20:16.566474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518891209537994:2079] [node 6] ICS09 handshake done sender: [5:7571518891209537959:2094] self: [5:7571518891209537958:0] peer: [6:7571518890229430380:0] socket: 423 qp: -1
2025-11-11T17:20:16.566513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518891209537994:2079] [node 6] ICS10 traffic start
2025-11-11T17:20:16.566554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:16.568048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209537996:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.573916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518891209537989:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.574082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518886914570615:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.574272Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518891209537976:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.574496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518886914570614:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.616322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209538000:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.635629Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518891209537996:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.650232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209538002:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.655457Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518891209537976:2105] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:16.657185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518886914570613:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.677144Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:16.681675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518886914570614:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.682252Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518891209537989:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:16.689738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:16.689802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518886914570614:2049] [node 2] ICP22 created new session: [5:7571518891209538004:2049]
2025-11-11T17:20:16.689860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518891209538004:2049] [node 2] ICS09 handshake done sender: [5:7571518891209537976:2105] self: [5:7571518891209537999:0] peer: [2:7571518890031514159:0] socket: 430 qp: -1
2025-11-11T17:20:16.689906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518891209538004:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:16.689943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:16.698336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518886914570615:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518891209537989:2117] held: no
2025-11-11T17:20:16.698445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518886914570615:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.698546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:16.698600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518886914570615:2050] [node 3] ICP22 created new session: [5:7571518891209538006:2050]
2025-11-11T17:20:16.698646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518891209538006:2050] [node 3] ICS09 handshake done sender: [5:7571518891209537989:2117] self: [5:7571518891209537998:0] peer: [3:7571518889095083540:0] socket: 454 qp: -1
2025-11-11T17:20:16.698680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518891209538006:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:16.698747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:16.746967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209538001:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.761571Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518891209537967:2097] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:16.781784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518891209538009:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:334
2025-11-11T17:20:16.793854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518891209537971:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.793976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:16.794037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518891209537971:2100] [node 4] ICP22 created new session: [5:7571518891209538010:2100]
2025-11-11T17:20:16.794096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518891209538010:2100] [node 4] ICS09 handshake done sender: [5:7571518891209537967:2097] self: [5:7571518891209537992:0] peer: [4:7571518890281824323:0] socket: 421 qp: -1
2025-11-11T17:20:16.794139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518891209538010:2100] [node 4] ICS10 traffic start
2025-11-11T17:20:16.794204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:16.798788Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:16.802776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518886914570613:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518891209537996:2119] held: no
2025-11-11T17:20:16.802964Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518891209537996:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:16.824422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518886914570613:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.709782Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518890031514171:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:16.734080Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514182:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.749947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518885736546794:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.757673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518890031514173:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.757717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518890031514173:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.773787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:16.790441Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518890031514173:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:16.791592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518890031514153:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.791709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:16.791755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518890031514153:2108] [node 6] ICP22 created new session: [2:7571518890031514192:2108]
2025-11-11T17:20:16.791804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518890031514192:2108] [node 6] ICS09 handshake done sender: [2:7571518890031514173:2117] self: [2:7571518890031514172:0] peer: [6:7571518890229430403:0] socket: 456 qp: -1
2025-11-11T17:20:16.791843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518890031514192:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:16.791874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:16.795942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518885736546794:2049] [node 3] ICP22 created new session: [2:7571518890031514191:2049]
2025-11-11T17:20:16.824275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518890031514191:2049] [node 3] ICS09 handshake done sender: [2:7571518890031514171:2116] self: [2:7571518890031514166:0] peer: [3:7571518889095083566:0] socket: 419 qp: -1
2025-11-11T17:20:16.826674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518890031514191:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:16.844092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:16.646145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518890229430397:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:16.757696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518889095083562:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.774972Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518889095083562:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:16.775834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518889095083542:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.775928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:16.775977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518889095083542:2109] [node 4] ICP22 created new session: [3:7571518889095083569:2109]
2025-11-11T17:20:16.776038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518889095083569:2109] [node 4] ICS09 handshake done sender: [3:7571518889095083562:2120] self: [3:7571518889095083558:0] peer: [4:7571518890281824353:0] socket: 478 qp: -1
2025-11-11T17:20:16.776073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518889095083569:2109] [node 4] ICS10 traffic start
2025-11-11T17:20:16.776112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:16.790617Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:16.800155Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518889095083557:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:16.810147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518884800116173:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518889095083557:2118] held: no
2025-11-11T17:20:16.838167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518884800116173:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24605864ns) {HandleHandshakeStatus:395(24602062ns) {LogHandshakeFail:472(17926ns) {UpdateErrorStateLog:544(5993ns)} DropOutgoingHandshake:456(3146ns) ProcessPendingSessionEvents:495(12890866ns) {DropSessionEvent:512(24203ns) DropSessionEvent:512(10999ns) DropSessionEvent:512(5193ns) DropSessionEvent:512(4984ns) DropSessionEvent:512(11479315ns) DropSessionEvent:512(12275ns) DropSessionEvent:512(1323889ns) DropSessionEvent:512(10818ns) DropSessionEvent:512(5239ns) DropSessionEvent:512(4127ns)}}}
2025-11-11T17:20:16.838227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518884800116173:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.838272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:16.838329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518884800116173:2049] [node 2] ICP22 created new session: [3:7571518889095083571:2049]
2025-11-11T17:20:16.838375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518889095083571:2049] [node 2] ICS09 handshake done sender: [3:7571518889095083557:2118] self: [3:7571518889095083566:0] peer: [2:7571518890031514166:0] socket: 459 qp: -1
2025-11-11T17:20:16.838414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518889095083571:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:16.838456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:16.857946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:16.872231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518886914570613:2048] [node 1] ICP22 created new session: [5:7571518891209538012:2048]
2025-11-11T17:20:16.877762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518886914570614:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(123046300ns) {HandleHandshakeStatus:329(123043845ns) {DropIncomingHandshake:441(9411ns) DropOutgoingHandshake:456(30236ns) SwitchToState:178(974ns) ProcessPendingSessionEvents:495(78040511ns) {ForwardSessionEventToSession:612(27893678ns) ForwardSessionEventToSession:612(4359483ns) ForwardSessionEventToSession:612(5289292ns) ForwardSessionEventToSession:612(4701ns) ForwardSessionEventToSession:612(4013ns) ForwardSessionEventToSession:612(18301300ns) ForwardSessionEventToSession:612(5455ns) ForwardSessionEventToSession:612(9474945ns) ForwardSessionEventToSession:612(5289ns) ForwardSessionEventToSession:612(12670498ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:16.854008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518885986856961:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(124037179ns) {HandleHandshakeStatus:329(124034454ns) {DropIncomingHandshake:441(991ns) DropOutgoingHandshake:456(5975ns) SwitchToState:178(1210ns) ProcessPendingSessionEvents:495(15821961ns) {ForwardSessionEventToSession:612(15753599ns) ForwardSessionEventToSession:612(7372ns) ForwardSessionEventToSession:612(7273ns) ForwardSessionEventToSession:612(6296ns) ForwardSessionEventToSession:612(2713ns) ForwardSessionEventToSession:612(9574ns) ForwardSessionEventToSession:612(3695ns) ForwardSessionEventToSession:612(2575ns) ForwardSessionEventToSession:612(13306ns) ForwardSessionEventToSession:612(2309ns)}}}
2025-11-11T17:20:16.491320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:16.646199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:16.653909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518890229430394:2072] [node 5] ICS09 handshake done sender: [6:7571518890229430332:2074] self: [6:7571518890229430380:0] peer: [5:7571518891209537958:0] socket: 433 qp: -1
2025-11-11T17:20:16.491366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518885381755938:2048] [node 2] ICP22 created new session: [1:7571518889676723323:2048]
2025-11-11T17:20:16.491406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518889676723323:2048] [node 2] ICS09 handshake done sender: [1:7571518889676723299:2110] self: [1:7571518889676723298:0] peer: [2:7571518890031514168:0] socket: 418 qp: -1
2025-11-11T17:20:16.491440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518889676723323:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:16.491470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:16.526944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518889676723302:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:16.531391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518889676723328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3142
2025-11-11T17:20:16.578030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723328:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.578162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518885381755938:2048] [node 2] ICP18 handshake (actor [1:7571518889676723328:2119]) is too old
2025-11-11T17:20:16.589820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518889676723300:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.616942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723312:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.616984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518889676723312:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.641740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:16.641792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518889676723300:2109] [node 6] ICP22 created new session: [1:7571518889676723331:2109]
2025-11-11T17:20:16.642119Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518889676723312:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:16.662255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518889676723331:2109] [node 6] ICS09 handshake done sender: [1:7571518889676723302:2111] self: [1:7571518889676723301:0] peer: [6:7571518890229430384:0] socket: 424 qp: -1
2025-11-11T17:20:16.670397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518889676723309:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.673753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518889676723331:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:16.689782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:16.690341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518889676723300:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(111378117ns) {HandleHandshakeStatus:329(100304603ns) {DropIncomingHandshake:441(20765ns) DropOutgoingHandshake:456(3865ns) SwitchToState:178(496ns) ProcessPendingSessionEvents:495(162099ns) {ForwardSessionEventToSession:612(51834ns) ForwardSessionEventToSession:612(3760ns) ForwardSessionEventToSession:612(67602ns) ForwardSessionEventToSession:612(5335ns) ForwardSessionEventToSession:612(3178ns) ForwardSessionEventToSession:612(5341ns) ForwardSessionEventToSession:612(2884ns) ForwardSessionEventToSession:612(8509ns) ForwardSessionEventToSession:612(2556ns) ForwardSessionEventToSession:612(3428ns)}}}
2025-11-11T17:20:16.691336Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518889676723310:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.691374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518889676723310:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:16.709705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:16.721983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518889676723309:2108] [node 4] ICP22 created new session: [1:7571518889676723333:2108]
2025-11-11T17:20:16.722707Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518889676723310:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:16.753665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518889676723333:2108] [node 4] ICS09 handshake done sender: [1:7571518889676723312:2114] self: [1:7571518889676723311:0] peer: [4:7571518890281824342:0] socket: 451 qp: -1
2025-11-11T17:20:16.753729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518889676723333:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:16.891217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518891209538012:2048] [node 1] ICS09 handshake done sender: [5:7571518891209537996:2119] self: [5:7571518891209538003:0] peer: [1:7571518889676723308:0] socket: 437 qp: -1
2025-11-11T17:20:16.854114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518885986856961:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:16.653964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518890229430394:2072] [node 5] ICS10 traffic start
2025-11-11T17:20:16.653647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518885934463009:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(247497882ns) {Configure:104(247485710ns) {ProcessConfigured:135(26388401ns) {StartInitialHandshake:157(26384403ns) {DropHandshakes:469(3293ns) {DropIncomingHandshake:441(698ns) DropOutgoingHandshake:456(819ns)} GenerateSessionVirtualId:988(994ns) PrepareNewSessionHandshake:476(4172ns) {SwitchToState:178(1303ns)}}}}}
2025-11-11T17:20:16.662330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:16.667249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518890229430376:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.667707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430400:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.726275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518890229430374:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(110743167ns) {HandleHandshakeStatus:329(110740376ns) {DropIncomingHandshake:441(594ns) DropOutgoingHandshake:456(4719ns) SwitchToState:178(703ns) ProcessPendingSessionEvents:495(16075389ns) {ForwardSessionEventToSession:612(16011044ns) ForwardSessionEventToSession:612(7444ns) ForwardSessionEventToSession:612(8475ns) ForwardSessionEventToSession:612(9399ns) ForwardSessionEventToSession:612(3298ns) ForwardSessionEventToSession:612(9396ns) ForwardSessionEventToSession:612(3122ns) ForwardSessionEventToSession:612(2741ns) ForwardSessionEventToSession:612(5619ns) ForwardSessionEventToSession:612(3379ns)}}}
2025-11-11T17:20:16.726355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518890229430374:2072] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:16.738084Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518890229430400:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:16.738238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518885934463009:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:16.797885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430404:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.809818Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:16.829832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518885934463009:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518890229430400:2122] held: no
2025-11-11T17:20:16.830201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518885934463009:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19684890ns) {HandleHandshakeStatus:395(8196113ns) {LogHandshakeFail:472(17839ns) {UpdateErrorStateLog:544(4835ns)} DropOutgoingHandshake:456(3751ns) ProcessPendingSessionEvents:495(66540ns) {DropSessionEvent:512(13663ns) DropSessionEvent:512(4736ns) DropSessionEvent:512(4634ns) DropSessionEvent:512(10034ns) DropSessionEvent:512(4015ns) DropSessionEvent:512(3849ns) DropSessionEvent:512(5995ns) DropSessionEvent:512(4372ns) DropSessionEvent:512(4860ns) DropSessionEvent:512(6407ns)}}}
2025-11-11T17:20:16.849691Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518890229430400:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:16.870732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518885934463009:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.870885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:16.891307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518891209538012:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:16.891372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:16.891650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518886914570613:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(67125381ns) {HandleHandshakeStatus:329(67121471ns) {DropIncomingHandshake:441(755ns) DropOutgoingHandshake:456(584ns) SwitchToState:178(793ns) ProcessPendingSessionEvents:495(692ns)}}
2025-11-11T17:20:16.894295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518885736546794:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(149769698ns) {HandleHandshakeStatus:329(149767323ns) {DropIncomingHandshake:441(20053ns) DropOutgoingHandshake:456(440ns) SwitchToState:178(711ns) ProcessPendingSessionEvents:495(7042962ns) {ForwardSessionEventToSession:612(6979576ns) ForwardSessionEventToSession:612(6480ns) ForwardSessionEventToSession:612(6941ns) ForwardSessionEventToSession:612(3516ns) ForwardSessionEventToSession:612(9862ns) ForwardSessionEventToSession:612(5550ns) ForwardSessionEventToSession:612(4229ns) ForwardSessionEventToSession:612(3189ns) ForwardSessionEventToSession:612(6441ns) ForwardSessionEventToSession:612(4114ns)}}}
2025-11-11T17:20:16.758338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518885381755999:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:16.758461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:16.758514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518885381755999:2107] [node 5] ICP22 created new session: [1:7571518889676723335:2107]
2025-11-11T17:20:16.758560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518889676723335:2107] [node 5] ICS09 handshake done sender: [1:7571518889676723310:2113] self: [1:7571518889676723308:0] peer: [5:7571518891209538003:0] socket: 461 qp: -1
2025-11-11T17:20:16.758604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518889676723335:2107] [node 5] ICS10 traffic start
2025-11-11T17:20:16.758671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:16.765898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:16.766397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518889676723309:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(95773863ns) {HandleHandshakeStatus:329(95770778ns) {DropIncomingHandshake:441(18831ns) DropOutgoingHandshake:456(456ns) SwitchToState:178(862ns) ProcessPendingSessionEvents:495(108704ns) {ForwardSessionEventToSession:612(57330ns) ForwardSessionEventToSession:612(3397ns) ForwardSessionEventToSession:612(2942ns) ForwardSessionEventToSession:612(8498ns) ForwardSessionEventToSession:612(2696ns) ForwardSessionEventToSession:612(7671ns) ForwardSessionEventToSession:612(3754ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(8003ns) ForwardSessionEventToSession:612(2953ns)}}}
2025-11-11T17:20:16.882033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518885986857019:2103] [node 6] ICP35 event processing took too much time PendingConnection:278(139385212ns) {HandleHandshakeStatus:329(139382995ns) {DropIncomingHandshake:441(776ns) DropOutgoingHandshake:456(533ns) SwitchToState:178(505ns) ProcessPendingSessionEvents:495(56147030ns) {ForwardSessionEventToSession:612(17482436ns) ForwardSessionEventToSession:612(7961ns) ForwardSessionEventToSession:612(6640144ns) ForwardSessionEventToSession:612(6230ns) ForwardSessionEventToSession:612(2875ns) ForwardSessionEventToSession:612(8224289ns) ForwardSessionEventToSession:612(19689432ns) ForwardSessionEventToSession:612(4045887ns) ForwardSessionEventToSession:612(8040ns) ForwardSessionEventToSession:612(3122ns)}}}
2025-11-11T17:20:16.882154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518885986857019:2103] [node 6] ICP18 handshake (actor [4:7571518890281824343:2118]) is too old
2025-11-11T17:20:16.870942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518885934463009:2049] [node 2] ICP22 created new session: [6:7571518890229430405:2049]
2025-11-11T17:20:16.870996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518890229430405:2049] [node 2] ICS09 handshake done sender: [6:7571518890229430400:2122] self: [6:7571518890229430403:0] peer: [2:7571518890031514172:0] socket: 479 qp: -1
2025-11-11T17:20:16.871043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518890229430405:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:16.871090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:16.890662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518890229430402:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:118
2025-11-11T17:20:16.913720Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12925) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:16.917102Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518890229430390:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:16.923071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518890229430376:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12925) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518890229430390:2119] held: no
2025-11-11T17:20:16.929663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518890229430376:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:16.929723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:16.933739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518890229430376:2070] [node 4] ICP22 created new session: [6:7571518890229430407:2070]
2025-11-11T17:20:16.942409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518890229430407:2070] [node 4] ICS09 handshake done sender: [6:7571518890229430390:2119] self: [6:7571518890229430401:0] peer: [4:7571518890281824331:0] socket: 435 qp: -1
2025-11-11T17:20:16.942479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518890229430407:2070] [node 4] ICS10 traffic start
2025-11-11T17:20:16.942534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:16.942826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518890229430376:2070] [node 4] ICP35 event processing took too much time PendingConnection:278(19286785ns) {HandleHandshakeStatus:329(19284057ns) {DropIncomingHandshake:441(2037ns) DropOutgoingHandshake:456(387ns) SwitchToState:178(1059ns) ProcessPendingSessionEvents:495(1084ns)}}
2025-11-11T17:20:17.012768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:17.013015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518890031514181:2048] [node 1] ICS07 socket disconnect 420 reason# EndOfStream
2025-11-11T17:20:17.013045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518890031514181:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.013132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:17.013169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518890031514181:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:17.019915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:17.020178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518889095083551:2048] [node 1] ICS01 socket: 431 reason# EndOfStream
2025-11-11T17:20:17.020263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518884800116172:2048] [node 1] ICP30 unregister session Session# [3:7571518889095083551:2048] VirtualId# [3:7571518889095083549:0]
2025-11-11T17:20:17.020302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518889095083551:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.020402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:17.020448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:17.029512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:17.043754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:17.044030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518890281824351:2048] [node 1] ICS01 socket: 452 reason# EndOfStream
2025-11-11T17:20:17.044079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518885986856961:2048] [node 1] ICP30 unregister session Session# [4:7571518890281824351:2048] VirtualId# [4:7571518890281824342:0]
2025-11-11T17:20:17.044114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518890281824351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.044196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:17.044247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:17.033651Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:20:17.039879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518885736546793:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:20:17.029768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518891209538012:2048] [node 1] ICS01 socket: 437 reason# EndOfStream
2025-11-11T17:20:17.045890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518890031514181:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:17.029880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518886914570613:2048] [node 1] ICP30 unregister session Session# [5:7571518891209538012:2048] VirtualId# [5:7571518891209538003:0]
2025-11-11T17:20:17.029918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518891209538012:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.029993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:17.030033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:17.061760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:17.057980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [2:7571518885736546793:2048] [node 1] ICP35 event processing took too much time StateWork:293(22528311ns) {HandleHandshakeStatus:395(22524621ns) {LogHandshakeFail:472(10498ns) {UpdateErrorStateLog:544(2520ns)} DropOutgoingHandshake:456(2586ns) ProcessPendingSessionEvents:495(431ns) StartResumeHandshake:172(11836745ns) {DropOutgoingHandshake:456(479ns)}}}
2025-11-11T17:20:17.067159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518890229430397:2048] [node 1] ICS01 socket: 434 reason# EndOfStream
2025-11-11T17:20:17.076638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518885934463008:2048] [node 1] ICP30 unregister session Session# [6:7571518890229430397:2048] VirtualId# [6:7571518890229430384:0]
2025-11-11T17:20:17.085913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518890229430397:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.086053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:17.086114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:17.087571Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Couldn't connect to any resolved address
2025-11-11T17:20:17.087865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518885736546793:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:17.087919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518890031514181:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:17.087947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518885736546793:2048] [node 1] ICP30 unregister session Session# [2:7571518890031514181:2048] VirtualId# [2:7571518890031514168:0]
2025-11-11T17:20:17.087993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:17.088027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518885736546793:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:17.088043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17932) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:17.099764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518885736546793:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:17.203380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:17.203623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518889095083571:2049] [node 2] ICS01 socket: 459 reason# EndOfStream
2025-11-11T17:20:17.203673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518884800116173:2049] [node 2] ICP30 unregister session Session# [3:7571518889095083571:2049] VirtualId# [3:7571518889095083566:0]
2025-11-11T17:20:17.203712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518889095083571:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.203801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:17.203843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:17.208079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:17.208334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518890281824339:2049] [node 2] ICS01 socket: 440 reason# EndOfStream
2025-11-11T17:20:17.208379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518885986856962:2049] [node 2] ICP30 unregister session Session# [4:7571518890281824339:2049] VirtualId# [4:7571518890281824335:0]
2025-11-11T17:20:17.208412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518890281824339:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.208480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:17.208523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:17.211233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:17.211468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518890229430405:2049] [node 2] ICS01 socket: 479 reason# EndOfStream
2025-11-11T17:20:17.211508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518885934463009:2049] [node 2] ICP30 unregister session Session# [6:7571518890229430405:2049] VirtualId# [6:7571518890229430403:0]
2025-11-11T17:20:17.211540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518890229430405:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.211625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:17.211658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:17.217752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:17.217977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518891209538004:2049] [node 2] ICS07 socket disconnect 430 reason# EndOfStream
2025-11-11T17:20:17.218024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518891209538004:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.218127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:17.218172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518891209538004:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:17.219025Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Couldn't connect to any resolved address
2025-11-11T17:20:17.219267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518886914570614:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:17.219311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518891209538004:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:17.219343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518886914570614:2049] [node 2] ICP30 unregister session Session# [5:7571518891209538004:2049] VirtualId# [5:7571518891209537999:0]
2025-11-11T17:20:17.219394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:17.219427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518886914570614:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:17.219444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:17.237738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518886914570614:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:17.285834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:17.286105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518890229430386:2050] [node 3] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-11T17:20:17.286140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518890229430386:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.286227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:17.286272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518890229430386:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:17.286800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:17.287014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518890281824357:2050] [node 3] ICS01 socket: 415 reason# EndOfStream
2025-11-11T17:20:17.287055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518885986856963:2050] [node 3] ICP30 unregister session Session# [4:7571518890281824357:2050] VirtualId# [4:7571518890281824353:0]
2025-11-11T17:20:17.287087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518890281824357:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.287152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:17.287191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:17.287280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:17.287512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518891209538006:2050] [node 3] ICS01 socket: 454 reason# EndOfStream
2025-11-11T17:20:17.287540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518886914570615:2050] [node 3] ICP30 unregister session Session# [5:7571518891209538006:2050] VirtualId# [5:7571518891209537998:0]
2025-11-11T17:20:17.287580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518891209538006:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.287633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:17.287670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:17.289059Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26886) Couldn't connect to any resolved address
2025-11-11T17:20:17.289306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518885934463010:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26886) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:17.289359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518890229430386:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:17.289392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518885934463010:2050] [node 3] ICP30 unregister session Session# [6:7571518890229430386:2050] VirtualId# [6:7571518890229430377:0]
2025-11-11T17:20:17.289436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:17.289485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518885934463010:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:17.289501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:17.300973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518885934463010:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:20:17.420664Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:17.420909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518890229430407:2070] [node 4] ICS01 socket: 435 reason# EndOfStream
2025-11-11T17:20:17.420956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518890229430376:2070] [node 4] ICP30 unregister session Session# [6:7571518890229430407:2070] VirtualId# [6:7571518890229430401:0]
2025-11-11T17:20:17.421120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:17.421314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518891209538010:2100] [node 4] ICS01 socket: 421 reason# EndOfStream
2025-11-11T17:20:17.421342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518891209537971:2100] [node 4] ICP30 unregister session Session# [5:7571518891209538010:2100] VirtualId# [5:7571518891209537992:0]
2025-11-11T17:20:17.421371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518891209538010:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.421495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:17.421543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:17.422019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518890229430407:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.422104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:17.422153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:17.508414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:17.508674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518890229430394:2072] [node 5] ICS01 socket: 433 reason# EndOfStream
2025-11-11T17:20:17.508720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518890229430374:2072] [node 5] ICP30 unregister session Session# [6:7571518890229430394:2072] VirtualId# [6:7571518890229430380:0]
2025-11-11T17:20:17.508760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518890229430394:2072] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:17.508880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:17.508928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:17.689915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518893684087046:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:17.689963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518893684087047:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:17.716620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518892442920602:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:17.716663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518892442920603:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:17.740174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518892082763991:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:17.740231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518892082763992:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:17.783933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518893390456033:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:17.783975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518893390456034:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:17.784006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518893390456035:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:17.830251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518893607256161:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:17.830322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518893607256162:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:17.832033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518893607256160:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:17.843056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518893355444958:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:17.843109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518893355444959:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:17.843152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518893355444960:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:18.158817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518897902223508:2098] [node 4] ICP01 ready to work
2025-11-11T17:20:18.153907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518897650412283:2072] [node 4] ICP01 ready to work
2025-11-11T17:20:18.242084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518896377731350:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:18.171882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518897650412282:2067] [node 5] ICP01 ready to work
2025-11-11T17:20:18.261710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518896737887961:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:18.273658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518897685423393:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:18.273645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518897979054413:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:18.279727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518896737887962:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.307824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518897902223508:2098] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(134592262ns) {Configure:104(134585275ns) {ProcessConfigured:135(115134ns) {StartInitialHandshake:157(112142ns) {DropHandshakes:469(4075ns) {DropIncomingHandshake:441(738ns) DropOutgoingHandshake:456(839ns)} GenerateSessionVirtualId:988(1482ns) PrepareNewSessionHandshake:476(2199ns) {SwitchToState:178(858ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.367191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412308:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.367269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412309:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.362026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518897685423395:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:18.365858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423396:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.371112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518897650412308:2096] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.371239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518893355444960:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.378272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518893390456033:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(197598148ns) {Configure:104(197594590ns) {ProcessConfigured:135(36148447ns) {StartInitialHandshake:157(36142061ns) {DropHandshakes:469(4292ns) {DropIncomingHandshake:441(929ns) DropOutgoingHandshake:456(819ns)} GenerateSessionVirtualId:988(2986ns) PrepareNewSessionHandshake:476(5079ns) {SwitchToState:178(1766ns)}}}}}
2025-11-11T17:20:18.313720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518897979054414:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:18.337896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518897902223514:2101] [node 6] ICP01 ready to work
2025-11-11T17:20:18.326027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518896377731355:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.293774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518896737887963:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:18.382012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518897650412283:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(139697574ns) {Configure:104(139692240ns) {ProcessConfigured:135(52659126ns) {StartInitialHandshake:157(52655516ns) {DropHandshakes:469(3063ns) {DropIncomingHandshake:441(638ns) DropOutgoingHandshake:456(662ns)} GenerateSessionVirtualId:988(950ns) PrepareNewSessionHandshake:476(5937ns) {SwitchToState:178(3041ns)}}}}}
2025-11-11T17:20:18.325180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518897979054415:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:18.335764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518896377731354:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24126
2025-11-11T17:20:18.370835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518896377731354:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.371644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518896377731357:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.439943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223527:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.371668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518896377731357:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.410238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412315:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.429636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423401:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.326437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518897979054413:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16448824ns) {Configure:104(16442177ns) {ProcessConfigured:135(67790ns) {StartInitialHandshake:157(64857ns) {DropHandshakes:469(6591ns) {DropIncomingHandshake:441(758ns) DropOutgoingHandshake:456(459ns)} GenerateSessionVirtualId:988(1073ns) PrepareNewSessionHandshake:476(1595ns) {SwitchToState:178(412ns)}}}}}
2025-11-11T17:20:18.372601Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518896377731357:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:18.372871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518896377731355:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.372936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:18.372985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518896377731355:2109] [node 6] ICP22 created new session: [3:7571518896377731362:2109]
2025-11-11T17:20:18.373033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518896377731362:2109] [node 6] ICS09 handshake done sender: [3:7571518896377731357:2112] self: [3:7571518896377731356:0] peer: [6:7571518897650412314:0] socket: 427 qp: -1
2025-11-11T17:20:18.373075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518896377731362:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:18.373109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:18.397699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518896377731359:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:18.449865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518897685423401:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.356357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518897979054416:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:117
2025-11-11T17:20:18.412545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518897979054420:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:117
2025-11-11T17:20:18.458063Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054416:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.482211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518897979054414:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(155451598ns) {Configure:104(155447862ns) {ProcessConfigured:135(98397ns) {StartInitialHandshake:157(87957ns) {DropHandshakes:469(2422ns) {DropIncomingHandshake:441(510ns) DropOutgoingHandshake:456(547ns)} GenerateSessionVirtualId:988(1041ns) PrepareNewSessionHandshake:476(1425ns) {SwitchToState:178(596ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.425906Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.490061Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518897902223527:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.426925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518893355444960:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518897650412308:2096] held: no
2025-11-11T17:20:18.427250Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518897650412308:2096] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:18.438271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518893355444960:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.438344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:18.438401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518893355444960:2050] [node 3] ICP22 created new session: [6:7571518897650412328:2050]
2025-11-11T17:20:18.438464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518897650412328:2050] [node 3] ICS09 handshake done sender: [6:7571518897650412308:2096] self: [6:7571518897650412314:0] peer: [3:7571518896377731356:0] socket: 430 qp: -1
2025-11-11T17:20:18.438502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518897650412328:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:18.438557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:18.493686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518897650412309:2097] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.495204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518893355444959:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(317200046ns) {Configure:104(317194397ns) {ProcessConfigured:135(81570ns) {StartInitialHandshake:157(76861ns) {DropHandshakes:469(4204ns) {DropIncomingHandshake:441(792ns) DropOutgoingHandshake:456(634ns)} GenerateSessionVirtualId:988(1081ns) PrepareNewSessionHandshake:476(2698ns) {SwitchToState:178(820ns)}}}}}
2025-11-11T17:20:18.522454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518892082763992:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(243971428ns) {Configure:104(243965010ns) {ProcessConfigured:135(26091285ns) {StartInitialHandshake:157(26088123ns) {DropHandshakes:469(2057ns) {DropIncomingHandshake:441(546ns) DropOutgoingHandshake:456(232ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:476(5108ns) {SwitchToState:178(2984ns)}}}}}
2025-11-11T17:20:18.451342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518893390456035:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(269472161ns) {Configure:104(269460537ns) {ProcessConfigured:135(57867ns) {StartInitialHandshake:157(55682ns) {DropHandshakes:469(2172ns) {DropIncomingHandshake:441(567ns) DropOutgoingHandshake:456(347ns)} GenerateSessionVirtualId:988(725ns) PrepareNewSessionHandshake:476(1271ns) {SwitchToState:178(439ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.554484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518896377731364:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24126
2025-11-11T17:20:18.517673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054420:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.490279Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518897685423396:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.514532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223529:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.562046Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518896377731364:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.565183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518893684087046:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(415178006ns) {Configure:104(415167708ns) {ProcessConfigured:135(3246577ns) {StartInitialHandshake:157(3244077ns) {DropHandshakes:469(2822ns) {DropIncomingHandshake:441(620ns) DropOutgoingHandshake:456(349ns)} GenerateSessionVirtualId:988(631ns) PrepareNewSessionHandshake:476(3181ns) {SwitchToState:178(1262ns)}}}}}
2025-11-11T17:20:18.581968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518896737887963:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(216818357ns) {Configure:104(216805243ns) {ProcessConfigured:135(78798ns) {StartInitialHandshake:157(74911ns) {DropHandshakes:469(2223ns) {DropIncomingHandshake:441(526ns) DropOutgoingHandshake:456(551ns)} GenerateSessionVirtualId:988(603ns) PrepareNewSessionHandshake:476(1029ns) {SwitchToState:178(237ns)}}}}}
2025-11-11T17:20:18.570159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518893684087047:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(296242589ns) {Configure:104(296231410ns) {ProcessConfigured:135(60524ns) {StartInitialHandshake:157(58336ns) {DropHandshakes:469(1973ns) {DropIncomingHandshake:441(498ns) DropOutgoingHandshake:456(538ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:476(1541ns) {SwitchToState:178(427ns)}}}}}
2025-11-11T17:20:18.609659Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054422:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.609724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518897979054422:2121] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.609736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518893390456034:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(380561563ns) {Configure:104(380556256ns) {ProcessConfigured:135(64645357ns) {StartInitialHandshake:157(64642230ns) {DropHandshakes:469(741ns) {DropIncomingHandshake:441(228ns) DropOutgoingHandshake:456(99ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(9221ns) {SwitchToState:178(4956ns)}}}}}
2025-11-11T17:20:18.610778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423409:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.556263Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518897902223529:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.619832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518896377731371:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24126
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.586422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412336:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.558369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518893607256161:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(273447541ns) {Configure:104(268067077ns) {ProcessConfigured:135(91472ns) {StartInitialHandshake:157(85029ns) {DropHandshakes:469(3897ns) {DropIncomingHandshake:441(817ns) DropOutgoingHandshake:456(1021ns)} GenerateSessionVirtualId:988(1863ns) PrepareNewSessionHandshake:476(2332ns) {SwitchToState:178(919ns)}}}}}
2025-11-11T17:20:18.582855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518896737887966:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8813
2025-11-11T17:20:18.640759Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518896377731352:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.640814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518896377731352:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.664058Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518896377731352:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:18.664428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518896377731350:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.664485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:18.664536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518896377731350:2108] [node 5] ICP22 created new session: [3:7571518896377731373:2108]
2025-11-11T17:20:18.664581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518896377731373:2108] [node 5] ICS09 handshake done sender: [3:7571518896377731352:2110] self: [3:7571518896377731351:0] peer: [5:7571518897902223538:0] socket: 421 qp: -1
2025-11-11T17:20:18.664615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518896377731373:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:18.664652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:18.635639Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518897979054422:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:18.662607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518897979054435:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.583008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518896737887967:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8813
2025-11-11T17:20:18.583055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518896737887968:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8813
2025-11-11T17:20:18.583146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518896737887969:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8813
2025-11-11T17:20:18.594624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518896737887962:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(268674173ns) {Configure:104(268664092ns) {ProcessConfigured:135(75689ns) {StartInitialHandshake:157(72674ns) {DropHandshakes:469(2155ns) {DropIncomingHandshake:441(489ns) DropOutgoingHandshake:456(364ns)} GenerateSessionVirtualId:988(1325ns) PrepareNewSessionHandshake:476(1937ns) {SwitchToState:178(1128ns)}}}}}
2025-11-11T17:20:18.594790Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887966:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.609668Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887969:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.618527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518892442920603:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(344607902ns) {Configure:104(344600123ns) {ProcessConfigured:135(82124ns) {StartInitialHandshake:157(79060ns) {DropHandshakes:469(4173ns) {DropIncomingHandshake:441(1127ns) DropOutgoingHandshake:456(888ns)} GenerateSessionVirtualId:988(1245ns) PrepareNewSessionHandshake:476(1822ns) {SwitchToState:178(663ns)}}}}}
2025-11-11T17:20:18.621881Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887967:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.669792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518896737887961:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(331905981ns) {Configure:104(331899801ns) {ProcessConfigured:135(15785064ns) {StartInitialHandshake:157(15782452ns) {DropHandshakes:469(2801ns) {DropIncomingHandshake:441(497ns) DropOutgoingHandshake:456(593ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:476(3881ns) {SwitchToState:178(1846ns)}}}}}
2025-11-11T17:20:18.570881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518893607256160:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.574699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518897902223514:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(160147600ns) {Configure:104(160140316ns) {ProcessConfigured:135(63576ns) {StartInitialHandshake:157(60875ns) {DropHandshakes:469(976ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(225ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:476(2167ns) {SwitchToState:178(602ns)}}}}}
2025-11-11T17:20:18.607184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223528:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.615119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518893607256162:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(324649100ns) {Configure:104(324640410ns) {ProcessConfigured:135(13175247ns) {StartInitialHandshake:157(13172112ns) {DropHandshakes:469(2125ns) {DropIncomingHandshake:441(378ns) DropOutgoingHandshake:456(421ns)} GenerateSessionVirtualId:988(789ns) PrepareNewSessionHandshake:476(5321ns) {SwitchToState:178(3181ns)}}}}}
2025-11-11T17:20:18.615327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518893607256162:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.630015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518897902223528:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.630365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518897902223508:2098] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.630626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223539:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.681684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887968:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.666095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518893355444958:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(334699966ns) {Configure:104(334691337ns) {ProcessConfigured:135(32761128ns) {StartInitialHandshake:157(32758100ns) {DropHandshakes:469(4365ns) {DropIncomingHandshake:441(1109ns) DropOutgoingHandshake:456(712ns)} GenerateSessionVirtualId:988(1282ns) PrepareNewSessionHandshake:476(3211ns) {SwitchToState:178(1495ns)}}}}}
2025-11-11T17:20:18.672406Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518896377731371:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.677221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518892082763991:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(318686173ns) {Configure:104(318679605ns) {ProcessConfigured:135(30524550ns) {StartInitialHandshake:157(30521175ns) {DropHandshakes:469(2408ns) {DropIncomingHandshake:441(409ns) DropOutgoingHandshake:456(565ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:476(3669ns) {SwitchToState:178(1183ns)}}}}}
2025-11-11T17:20:18.685907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518892082763992:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.686451Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887972:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.686490Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518896737887972:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.688272Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518896737887972:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:18.688562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518896737887962:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.688627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:18.688685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518896737887962:2107] [node 4] ICP22 created new session: [2:7571518896737887984:2107]
2025-11-11T17:20:18.688732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518896737887984:2107] [node 4] ICS09 handshake done sender: [2:7571518896737887972:2115] self: [2:7571518896737887971:0] peer: [4:7571518897685423411:0] socket: 452 qp: -1
2025-11-11T17:20:18.688763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518896737887984:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:18.688794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:18.672604Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518897685423409:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.690247Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887965:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.690287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518896737887965:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.675886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518897650412336:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.709732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887977:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.709801Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518896737887977:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.711318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518896737887988:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8813
2025-11-11T17:20:18.711518Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518896737887977:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.676454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518897685423393:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(374523807ns) {Configure:104(364414487ns) {ProcessConfigured:135(14395996ns) {StartInitialHandshake:157(14392943ns) {DropHandshakes:469(7807417ns) {DropIncomingHandshake:441(533ns) DropOutgoingHandshake:456(2835ns)} GenerateSessionVirtualId:988(2361ns) PrepareNewSessionHandshake:476(2049ns) {SwitchToState:178(45ns)}}}}}
2025-11-11T17:20:18.690391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518897979054434:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:117
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.729832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054434:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.737794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518897979054436:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.742249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518897979054415:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.742328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:18.742382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518897979054415:2108] [node 5] ICP22 created new session: [1:7571518897979054437:2108]
2025-11-11T17:20:18.742432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518897979054437:2108] [node 5] ICS09 handshake done sender: [1:7571518897979054422:2121] self: [1:7571518897979054421:0] peer: [5:7571518897902223530:0] socket: 438 qp: -1
2025-11-11T17:20:18.742470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518897979054437:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:18.742509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:18.718752Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.677107Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518897685423398:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.676037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518893355444959:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.711998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518896377731359:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(281821453ns) {Configure:104(281811780ns) {ProcessConfigured:135(13970564ns) {StartInitialHandshake:157(13967618ns) {DropHandshakes:469(1694ns) {DropIncomingHandshake:441(281ns) DropOutgoingHandshake:456(323ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:476(2411ns) {SwitchToState:178(371ns)}}}}}
2025-11-11T17:20:18.677138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518897685423398:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.682336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518893390456034:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.688160Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518897685423398:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:18.691949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518897685423395:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.692086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:18.692142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518897685423395:2109] [node 5] ICP22 created new session: [4:7571518897685423416:2109]
2025-11-11T17:20:18.692196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518897685423416:2109] [node 5] ICS09 handshake done sender: [4:7571518897685423398:2112] self: [4:7571518897685423397:0] peer: [5:7571518897902223540:0] socket: 432 qp: -1
2025-11-11T17:20:18.692245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518897685423416:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:18.692287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:18.697568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423415:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.727306Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518897685423409:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.758469Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054435:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.770035Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054418:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.697633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518893355444958:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.698685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518897650412282:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(379863486ns) {Configure:104(379853437ns) {ProcessConfigured:135(56846030ns) {StartInitialHandshake:157(56842744ns) {DropHandshakes:469(2618ns) {DropIncomingHandshake:441(562ns) DropOutgoingHandshake:456(726ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:476(3503ns) {SwitchToState:178(1734ns)}}}}}
2025-11-11T17:20:18.737214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412342:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.757923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412343:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.770532Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518897650412336:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.721839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518892442920602:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(384249142ns) {Configure:104(384239908ns) {ProcessConfigured:135(39935568ns) {StartInitialHandshake:157(39932328ns) {DropHandshakes:469(2439ns) {DropIncomingHandshake:441(767ns) DropOutgoingHandshake:456(230ns)} GenerateSessionVirtualId:988(550ns) PrepareNewSessionHandshake:476(4198ns) {SwitchToState:178(966ns)}}}}}
2025-11-11T17:20:18.765605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.744050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518896377731380:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24126
2025-11-11T17:20:18.720048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518897902223508:2098] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518897902223528:2113] held: no
2025-11-11T17:20:18.770081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518897979054418:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.781187Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054436:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.781310Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518897979054418:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:18.796447Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054429:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.796484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518897979054429:2123] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.738133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518892442920602:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.741884Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518896737887965:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:18.744874Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887988:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.744988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518896737887962:2107] [node 4] ICP18 handshake (actor [2:7571518896737887988:2119]) is too old
2025-11-11T17:20:18.773736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518892442920603:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.775522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518892442920602:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16243825ns) {IncomingHandshake:261(16240789ns) {DropIncomingHandshake:441(859ns) PrepareNewSessionHandshake:476(3058ns) {SwitchToState:178(772ns)} GenerateSessionVirtualId:988(902ns) IssueIncomingHandshakeReply:193(8367219ns) {SwitchToState:178(2146ns)}}}
2025-11-11T17:20:18.779130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518896737887963:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.779190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:18.779232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518896737887963:2109] [node 6] ICP22 created new session: [2:7571518896737887992:2109]
2025-11-11T17:20:18.779277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518896737887992:2109] [node 6] ICS09 handshake done sender: [2:7571518896737887965:2110] self: [2:7571518896737887964:0] peer: [6:7571518897650412340:0] socket: 450 qp: -1
2025-11-11T17:20:18.779314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518896737887992:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:18.779348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:18.782336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:18.782403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518892442920603:2049] [node 3] ICP22 created new session: [2:7571518896737887995:2049]
2025-11-11T17:20:18.789986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518896737887995:2049] [node 3] ICS09 handshake done sender: [2:7571518896737887977:2116] self: [2:7571518896737887976:0] peer: [3:7571518896377731375:0] socket: 455 qp: -1
2025-11-11T17:20:18.790038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518896737887995:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:18.790102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:18.791850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518892442920603:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(28575036ns) {HandleHandshakeStatus:329(16706289ns) {DropIncomingHandshake:441(20040ns) DropOutgoingHandshake:456(516ns) SwitchToState:178(534ns) ProcessPendingSessionEvents:495(122072ns) {ForwardSessionEventToSession:612(67583ns) ForwardSessionEventToSession:612(6087ns) ForwardSessionEventToSession:612(5952ns) ForwardSessionEventToSession:612(3131ns) ForwardSessionEventToSession:612(5268ns) ForwardSessionEventToSession:612(3021ns) ForwardSessionEventToSession:612(3435ns) ForwardSessionEventToSession:612(8923ns) ForwardSessionEventToSession:612(5587ns) ForwardSessionEventToSession:612(4622ns)}}}
2025-11-11T17:20:18.720316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223541:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.761706Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518896377731371:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.806080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518897979054413:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.811984Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518897979054429:2123] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:18.812249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518893684087046:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.812309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:18.812369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518893684087046:2048] [node 2] ICP22 created new session: [1:7571518897979054446:2048]
2025-11-11T17:20:18.812413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518897979054446:2048] [node 2] ICS09 handshake done sender: [1:7571518897979054429:2123] self: [1:7571518897979054428:0] peer: [2:7571518896737887991:0] socket: 445 qp: -1
2025-11-11T17:20:18.812443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518897979054446:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:18.812477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:18.817782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518893684087047:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(35849343ns) {IncomingHandshake:261(15957199ns) {DropIncomingHandshake:441(622ns) PrepareNewSessionHandshake:476(15901014ns) {SwitchToState:178(8540ns)} GenerateSessionVirtualId:988(803ns) IssueIncomingHandshakeReply:193(7632ns)}}
2025-11-11T17:20:18.793802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412344:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.790102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518892082763992:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.790169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:18.790215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518892082763992:2049] [node 2] ICP22 created new session: [3:7571518896377731381:2049]
2025-11-11T17:20:18.790275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518896377731381:2049] [node 2] ICS09 handshake done sender: [3:7571518896377731371:2116] self: [3:7571518896377731375:0] peer: [2:7571518896737887976:0] socket: 456 qp: -1
2025-11-11T17:20:18.793902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518896377731381:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:18.818587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518896377731382:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24126
2025-11-11T17:20:18.825175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:18.828008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518896377731382:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.828120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518892082763991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.771127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518893390456034:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.839197Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.817651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518893355444959:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.817728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:18.824862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518893355444959:2049] [node 2] ICP22 created new session: [6:7571518897650412345:2049]
2025-11-11T17:20:18.826437Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.827304Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518897650412343:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.827413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518893355444958:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518897650412309:2097] held: no
2025-11-11T17:20:18.827559Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518897650412309:2097] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:18.827738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518897650412282:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.833815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518897650412345:2049] [node 2] ICS09 handshake done sender: [6:7571518897650412336:2116] self: [6:7571518897650412340:0] peer: [2:7571518896737887964:0] socket: 451 qp: -1
2025-11-11T17:20:18.838139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518897650412345:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:18.845794Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.846098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:18.850173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518893355444958:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.850268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:18.850327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518893355444958:2048] [node 1] ICP22 created new session: [6:7571518897650412348:2048]
2025-11-11T17:20:18.850387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518897650412348:2048] [node 1] ICS09 handshake done sender: [6:7571518897650412309:2097] self: [6:7571518897650412341:0] peer: [1:7571518897979054417:0] socket: 431 qp: -1
2025-11-11T17:20:18.850430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518897650412348:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:18.850475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.858751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412351:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.720770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223542:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.797924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:18.864890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518893355444959:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(68474345ns) {HandleHandshakeStatus:329(68471660ns) {DropIncomingHandshake:441(2359ns) DropOutgoingHandshake:456(7222ns) SwitchToState:178(456ns) ProcessPendingSessionEvents:495(102949ns) {ForwardSessionEventToSession:612(57058ns) ForwardSessionEventToSession:612(3382ns) ForwardSessionEventToSession:612(7635ns) ForwardSessionEventToSession:612(3820ns) ForwardSessionEventToSession:612(5029ns) ForwardSessionEventToSession:612(3627ns) ForwardSessionEventToSession:612(3529ns) ForwardSessionEventToSession:612(4481ns) ForwardSessionEventToSession:612(3075ns) ForwardSessionEventToSession:612(3027ns)}}}
2025-11-11T17:20:18.865004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518893355444959:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.833956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:18.876503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412352:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.877776Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518897650412351:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.877866Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518897650412343:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:18.882216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518897650412282:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.882282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:18.882331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518897650412282:2067] [node 5] ICP22 created new session: [6:7571518897650412353:2067]
2025-11-11T17:20:18.882377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518897650412353:2067] [node 5] ICS09 handshake done sender: [6:7571518897650412343:2120] self: [6:7571518897650412346:0] peer: [5:7571518897902223534:0] socket: 477 qp: -1
2025-11-11T17:20:18.882413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518897650412353:2067] [node 5] ICS10 traffic start
2025-11-11T17:20:18.882467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.720961Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518897902223529:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:18.899811Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10032) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.901963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518897650412282:2067] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.848874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518892442920602:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518896737887969:2114] held: no
2025-11-11T17:20:18.814404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518893390456034:2049] [node 2] ICP22 created new session: [4:7571518897685423420:2049]
2025-11-11T17:20:18.838126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518897979054413:2109] [node 6] ICP22 created new session: [1:7571518897979054448:2109]
2025-11-11T17:20:18.913740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518897650412283:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.913915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518897650412283:2072] [node 4] ICP35 event processing took too much time PendingConnection:278(19970557ns) {IncomingHandshake:261(19966854ns) {DropIncomingHandshake:441(617ns) PrepareNewSessionHandshake:476(1480ns) {SwitchToState:178(458ns)} GenerateSessionVirtualId:988(665ns) IssueIncomingHandshakeReply:193(11956256ns) {SwitchToState:178(1765ns)}}}
2025-11-11T17:20:18.841893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518897685423420:2049] [node 2] ICS09 handshake done sender: [4:7571518897685423409:2117] self: [4:7571518897685423411:0] peer: [2:7571518896737887971:0] socket: 453 qp: -1
2025-11-11T17:20:18.842007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518897685423419:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.842902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423422:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.726004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518893607256160:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.726105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:18.726174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518893607256160:2048] [node 1] ICP22 created new session: [5:7571518897902223543:2048]
2025-11-11T17:20:18.726249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518897902223543:2048] [node 1] ICS09 handshake done sender: [5:7571518897902223529:2114] self: [5:7571518897902223530:0] peer: [1:7571518897979054421:0] socket: 422 qp: -1
2025-11-11T17:20:18.726294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518897902223543:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:18.726339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:18.732409Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518897902223527:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:18.732741Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518897902223528:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:18.743415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518893607256162:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.743506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:18.743565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518893607256162:2050] [node 3] ICP22 created new session: [5:7571518897902223546:2050]
2025-11-11T17:20:18.743616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518897902223546:2050] [node 3] ICS09 handshake done sender: [5:7571518897902223527:2112] self: [5:7571518897902223538:0] peer: [3:7571518896377731351:0] socket: 426 qp: -1
2025-11-11T17:20:18.743645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518897902223546:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:18.744252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518897902223508:2098] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.744287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:18.744330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518897902223508:2098] [node 4] ICP22 created new session: [5:7571518897902223547:2098]
2025-11-11T17:20:18.744372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518897902223547:2098] [node 4] ICS09 handshake done sender: [5:7571518897902223528:2113] self: [5:7571518897902223540:0] peer: [4:7571518897685423397:0] socket: 439 qp: -1
2025-11-11T17:20:18.744401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518897902223547:2098] [node 4] ICS10 traffic start
2025-11-11T17:20:18.744454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:18.746081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:18.801987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223550:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.842225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518897902223550:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.842354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518893607256161:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.853230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223552:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.859856Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518897902223535:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.859899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518897902223535:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.875224Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518897902223552:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.876064Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518897902223535:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:18.876895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518897902223514:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.876983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:18.877036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518897902223514:2101] [node 6] ICP22 created new session: [5:7571518897902223555:2101]
2025-11-11T17:20:18.877098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518897902223555:2101] [node 6] ICS09 handshake done sender: [5:7571518897902223535:2116] self: [5:7571518897902223534:0] peer: [6:7571518897650412346:0] socket: 476 qp: -1
2025-11-11T17:20:18.877129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518897902223555:2101] [node 6] ICS10 traffic start
2025-11-11T17:20:18.877165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:18.898344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518896377731385:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24126
2025-11-11T17:20:18.851556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518896737887997:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8813
2025-11-11T17:20:18.853362Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054432:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.853417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518897979054432:2124] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.867145Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518897979054432:2124] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:18.867441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518893684087047:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.867513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:18.867574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518893684087047:2049] [node 3] ICP22 created new session: [1:7571518897979054450:2049]
2025-11-11T17:20:18.867613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518897979054450:2049] [node 3] ICS09 handshake done sender: [1:7571518897979054432:2124] self: [1:7571518897979054431:0] peer: [3:7571518896377731384:0] socket: 420 qp: -1
2025-11-11T17:20:18.867649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518897979054450:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:18.867685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:18.873708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518897979054448:2109] [node 6] ICS09 handshake done sender: [1:7571518897979054418:2119] self: [1:7571518897979054417:0] peer: [6:7571518897650412341:0] socket: 428 qp: -1
2025-11-11T17:20:18.878864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518897979054448:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:18.881985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518897979054426:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.882014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518897979054426:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.899762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:18.918112Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518897979054426:2122] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.948391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518897650412356:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52307
2025-11-11T17:20:18.948947Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518897650412351:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:18.843387Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518897685423422:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.970335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518897979054414:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.970417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:18.970468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518897979054414:2107] [node 4] ICP22 created new session: [1:7571518897979054454:2107]
2025-11-11T17:20:18.970523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518897979054454:2107] [node 4] ICS09 handshake done sender: [1:7571518897979054426:2122] self: [1:7571518897979054425:0] peer: [4:7571518897685423425:0] socket: 446 qp: -1
2025-11-11T17:20:18.970561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518897979054454:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:18.970596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:18.846981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518897685423420:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:18.905955Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518896377731377:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.859818Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518896737887969:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:18.905999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518896377731377:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.937718Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518896377731382:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:18.942158Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.942604Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518896377731377:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:18.954913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518896377731359:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.954973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:18.955025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518896377731359:2107] [node 4] ICP22 created new session: [3:7571518896377731387:2107]
2025-11-11T17:20:18.955070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518896377731387:2107] [node 4] ICS09 handshake done sender: [3:7571518896377731377:2117] self: [3:7571518896377731376:0] peer: [4:7571518897685423424:0] socket: 469 qp: -1
2025-11-11T17:20:18.955109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518896377731387:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:18.955145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:18.961684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518892082763991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518896377731382:2119] held: no
2025-11-11T17:20:18.962021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518892082763991:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19325693ns) {HandleHandshakeStatus:395(19322281ns) {LogHandshakeFail:472(16149ns) {UpdateErrorStateLog:544(5800ns)} DropOutgoingHandshake:456(2888ns) ProcessPendingSessionEvents:495(99319ns) {DropSessionEvent:512(30409ns) DropSessionEvent:512(10864ns) DropSessionEvent:512(12783ns) DropSessionEvent:512(6017ns) DropSessionEvent:512(5319ns) DropSessionEvent:512(4399ns) DropSessionEvent:512(4594ns) DropSessionEvent:512(6625ns) DropSessionEvent:512(5715ns) DropSessionEvent:512(5867ns)}}}
2025-11-11T17:20:18.962066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518892082763991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.962106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:18.962151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518892082763991:2048] [node 1] ICP22 created new session: [3:7571518896377731389:2048]
2025-11-11T17:20:18.962190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518896377731389:2048] [node 1] ICS09 handshake done sender: [3:7571518896377731382:2119] self: [3:7571518896377731384:0] peer: [1:7571518897979054431:0] socket: 480 qp: -1
2025-11-11T17:20:18.962224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518896377731389:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:18.962252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:18.878311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518896737887980:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.878351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518896737887980:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.894405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518892442920602:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.894472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:18.894530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518892442920602:2048] [node 1] ICP22 created new session: [2:7571518896737887998:2048]
2025-11-11T17:20:18.947645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518897902223514:2101] [node 6] ICP35 event processing took too much time PendingConnection:278(70509033ns) {HandleHandshakeStatus:329(70507130ns) {DropIncomingHandshake:441(11544ns) DropOutgoingHandshake:456(473ns) SwitchToState:178(441ns) ProcessPendingSessionEvents:495(9624093ns) {ForwardSessionEventToSession:612(9548316ns) ForwardSessionEventToSession:612(6047ns) ForwardSessionEventToSession:612(13439ns) ForwardSessionEventToSession:612(2152ns) ForwardSessionEventToSession:612(1734ns) ForwardSessionEventToSession:612(4299ns) ForwardSessionEventToSession:612(3279ns) ForwardSessionEventToSession:612(4904ns) ForwardSessionEventToSession:612(5708ns) ForwardSessionEventToSession:612(23102ns)}}}
2025-11-11T17:20:18.894579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518896737887998:2048] [node 1] ICS09 handshake done sender: [2:7571518896737887969:2114] self: [2:7571518896737887991:0] peer: [1:7571518897979054428:0] socket: 449 qp: -1
2025-11-11T17:20:18.894615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518896737887998:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:18.894653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:18.949206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518897650412283:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:19.005906Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518896737887980:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:18.960326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:18.974914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518897650412283:2072] [node 4] ICP22 created new session: [6:7571518897650412357:2072]
2025-11-11T17:20:19.006728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518897650412357:2072] [node 4] ICS09 handshake done sender: [6:7571518897650412351:2122] self: [6:7571518897650412355:0] peer: [4:7571518897685423410:0] socket: 481 qp: -1
2025-11-11T17:20:19.006427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518897902223557:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12327
2025-11-11T17:20:18.974211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518897979054413:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(167913327ns) {HandleHandshakeStatus:329(167909851ns) {DropIncomingHandshake:441(18202ns) DropOutgoingHandshake:456(424ns) SwitchToState:178(367ns) ProcessPendingSessionEvents:495(13711975ns) {ForwardSessionEventToSession:612(13662003ns) ForwardSessionEventToSession:612(5915ns) ForwardSessionEventToSession:612(8756ns) ForwardSessionEventToSession:612(2565ns) ForwardSessionEventToSession:612(4334ns) ForwardSessionEventToSession:612(3931ns) ForwardSessionEventToSession:612(3882ns) ForwardSessionEventToSession:612(4357ns) ForwardSessionEventToSession:612(2822ns) ForwardSessionEventToSession:612(3154ns)}}}
2025-11-11T17:20:18.847034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:19.013749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518897650412357:2072] [node 4] ICS10 traffic start
2025-11-11T17:20:19.013827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:19.014337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518897650412283:2072] [node 4] ICP35 event processing took too much time PendingConnection:278(64893584ns) {HandleHandshakeStatus:329(64891303ns) {DropIncomingHandshake:441(655ns) DropOutgoingHandshake:456(5865ns) SwitchToState:178(522ns) ProcessPendingSessionEvents:495(115125ns) {ForwardSessionEventToSession:612(58925ns) ForwardSessionEventToSession:612(7807ns) ForwardSessionEventToSession:612(2965ns) ForwardSessionEventToSession:612(2811ns) ForwardSessionEventToSession:612(8190ns) ForwardSessionEventToSession:612(2292ns) ForwardSessionEventToSession:612(5816ns) ForwardSessionEventToSession:612(5804ns) ForwardSessionEventToSession:612(2452ns) ForwardSessionEventToSession:612(8427ns)}}}
2025-11-11T17:20:18.853808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518893390456035:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.862914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518893390456033:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:18.902124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518893390456034:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(140186693ns) {HandleHandshakeStatus:329(130823554ns) {DropIncomingHandshake:441(993ns) DropOutgoingHandshake:456(18431ns) SwitchToState:178(653ns) ProcessPendingSessionEvents:495(20085587ns) {ForwardSessionEventToSession:612(20033555ns) ForwardSessionEventToSession:612(6525ns) ForwardSessionEventToSession:612(3512ns) ForwardSessionEventToSession:612(8448ns) ForwardSessionEventToSession:612(3933ns) ForwardSessionEventToSession:612(3199ns) ForwardSessionEventToSession:612(4843ns) ForwardSessionEventToSession:612(4266ns) ForwardSessionEventToSession:612(5019ns) ForwardSessionEventToSession:612(2525ns)}}}
2025-11-11T17:20:18.902914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423426:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.915803Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518897685423412:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:18.915832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518897685423412:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:18.935533Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518897685423412:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:18.954593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518897685423428:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35647
2025-11-11T17:20:18.958244Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518897685423422:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:18.978112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518897685423393:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:18.978172Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.978182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:18.978252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518897685423393:2108] [node 6] ICP22 created new session: [4:7571518897685423429:2108]
2025-11-11T17:20:18.978303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518897685423429:2108] [node 6] ICS09 handshake done sender: [4:7571518897685423412:2118] self: [4:7571518897685423410:0] peer: [6:7571518897650412355:0] socket: 467 qp: -1
2025-11-11T17:20:18.978342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518897685423429:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:18.978400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:18.982845Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:18.988429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518893390456035:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518897685423419:2120] held: no
2025-11-11T17:20:18.988524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518893390456033:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.988603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:18.988656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518893390456033:2048] [node 1] ICP22 created new session: [4:7571518897685423431:2048]
2025-11-11T17:20:18.988701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518897685423431:2048] [node 1] ICS09 handshake done sender: [4:7571518897685423422:2121] self: [4:7571518897685423425:0] peer: [1:7571518897979054425:0] socket: 475 qp: -1
2025-11-11T17:20:18.988735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518897685423431:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:18.988764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:18.989023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518893390456033:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:18.994073Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518897685423419:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:18.998202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518893390456035:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:18.998251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:18.998299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518893390456035:2050] [node 3] ICP22 created new session: [4:7571518897685423433:2050]
2025-11-11T17:20:18.998340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518897685423433:2050] [node 3] ICS09 handshake done sender: [4:7571518897685423419:2120] self: [4:7571518897685423424:0] peer: [3:7571518896377731376:0] socket: 474 qp: -1
2025-11-11T17:20:18.998376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518897685423433:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:18.998408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:19.033687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518892082763992:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(147775188ns) {HandleHandshakeStatus:329(147772655ns) {DropIncomingHandshake:441(703ns) DropOutgoingHandshake:456(15007ns) SwitchToState:178(293ns) ProcessPendingSessionEvents:495(80111351ns) {ForwardSessionEventToSession:612(24063598ns) ForwardSessionEventToSession:612(11900134ns) ForwardSessionEventToSession:612(8045873ns) ForwardSessionEventToSession:612(19373ns) ForwardSessionEventToSession:612(19951210ns) ForwardSessionEventToSession:612(4956ns) ForwardSessionEventToSession:612(3493ns) ForwardSessionEventToSession:612(926674ns) ForwardSessionEventToSession:612(2157ns) ForwardSessionEventToSession:612(15163453ns)}}}
2025-11-11T17:20:19.037723Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518897902223550:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:19.041811Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:19.022220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518896737887961:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:19.022337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:19.022390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518896737887961:2108] [node 5] ICP22 created new session: [2:7571518901032855297:2108]
2025-11-11T17:20:19.022438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518901032855297:2108] [node 5] ICS09 handshake done sender: [2:7571518896737887980:2117] self: [2:7571518896737887975:0] peer: [5:7571518897902223551:0] socket: 424 qp: -1
2025-11-11T17:20:19.022480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518901032855297:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:19.022518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:19.050130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518893607256161:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518897902223550:2121] held: no
2025-11-11T17:20:19.050529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518893607256161:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:19.050572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:19.050626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518893607256161:2049] [node 2] ICP22 created new session: [5:7571518902197190854:2049]
2025-11-11T17:20:19.050674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518902197190854:2049] [node 2] ICS09 handshake done sender: [5:7571518897902223550:2121] self: [5:7571518897902223551:0] peer: [2:7571518896737887975:0] socket: 479 qp: -1
2025-11-11T17:20:19.050715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518902197190854:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:19.050775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:19.133956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:19.134204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518896737887998:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-11-11T17:20:19.134246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518892442920602:2048] [node 1] ICP30 unregister session Session# [2:7571518896737887998:2048] VirtualId# [2:7571518896737887991:0]
2025-11-11T17:20:19.134281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518896737887998:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.134380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:19.134438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:19.151203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:19.151495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518896377731389:2048] [node 1] ICS01 socket: 480 reason# EndOfStream
2025-11-11T17:20:19.151555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518892082763991:2048] [node 1] ICP30 unregister session Session# [3:7571518896377731389:2048] VirtualId# [3:7571518896377731384:0]
2025-11-11T17:20:19.151597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518896377731389:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.151703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:19.151747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:19.162803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:19.163014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518897685423431:2048] [node 1] ICS01 socket: 475 reason# EndOfStream
2025-11-11T17:20:19.163057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518893390456033:2048] [node 1] ICP30 unregister session Session# [4:7571518897685423431:2048] VirtualId# [4:7571518897685423425:0]
2025-11-11T17:20:19.163093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518897685423431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.163153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:19.163198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:19.161814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:19.162061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518897650412348:2048] [node 1] ICS01 socket: 431 reason# EndOfStream
2025-11-11T17:20:19.162106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518893355444958:2048] [node 1] ICP30 unregister session Session# [6:7571518897650412348:2048] VirtualId# [6:7571518897650412341:0]
2025-11-11T17:20:19.162139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518897650412348:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.162226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:19.162272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:19.169737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:19.170012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518897902223543:2048] [node 1] ICS07 socket disconnect 422 reason# EndOfStream
2025-11-11T17:20:19.170042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518897902223543:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.170169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:19.170216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518897902223543:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:19.173891Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Couldn't connect to any resolved address
2025-11-11T17:20:19.174234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518893607256160:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:19.174311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518897902223543:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:19.174367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518893607256160:2048] [node 1] ICP30 unregister session Session# [5:7571518897902223543:2048] VirtualId# [5:7571518897902223530:0]
2025-11-11T17:20:19.174443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:19.174496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518893607256160:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.174519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.185171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518893607256160:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:19.267166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:19.267426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518896377731381:2049] [node 2] ICS07 socket disconnect 456 reason# EndOfStream
2025-11-11T17:20:19.267460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518896377731381:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.267554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:19.267590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518896377731381:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:19.273779Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:20:19.274061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518892082763992:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:20:19.274098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518896377731381:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:19.273831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:19.274085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518897685423420:2049] [node 2] ICS07 socket disconnect 453 reason# EndOfStream
2025-11-11T17:20:19.274123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518897685423420:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.274220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:19.274276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518897685423420:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:19.274844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:19.274987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518897650412345:2049] [node 2] ICS01 socket: 451 reason# EndOfStream
2025-11-11T17:20:19.275015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518893355444959:2049] [node 2] ICP30 unregister session Session# [6:7571518897650412345:2049] VirtualId# [6:7571518897650412340:0]
2025-11-11T17:20:19.275039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518897650412345:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.275086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:19.274936Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address
2025-11-11T17:20:19.275127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:19.275202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518892082763992:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:19.275246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518896377731381:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:19.275278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518892082763992:2049] [node 2] ICP30 unregister session Session# [3:7571518896377731381:2049] VirtualId# [3:7571518896377731375:0]
2025-11-11T17:20:19.275325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:19.275347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518892082763992:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.275361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.275941Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address
2025-11-11T17:20:19.276228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518893390456034:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:19.276280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518897685423420:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:19.277690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:19.277905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518902197190854:2049] [node 2] ICS01 socket: 479 reason# EndOfStream
2025-11-11T17:20:19.277941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518893607256161:2049] [node 2] ICP30 unregister session Session# [5:7571518902197190854:2049] VirtualId# [5:7571518897902223551:0]
2025-11-11T17:20:19.277976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518902197190854:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.278082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:19.278127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:19.276328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518893390456034:2049] [node 2] ICP30 unregister session Session# [4:7571518897685423420:2049] VirtualId# [4:7571518897685423411:0]
2025-11-11T17:20:19.276363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:19.276388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518893390456034:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.276401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27938) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.283308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518893390456034:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:19.287813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518892082763992:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:19.361979Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:19.362211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518897902223546:2050] [node 3] ICS01 socket: 426 reason# EndOfStream
2025-11-11T17:20:19.362249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518893607256162:2050] [node 3] ICP30 unregister session Session# [5:7571518897902223546:2050] VirtualId# [5:7571518897902223538:0]
2025-11-11T17:20:19.362285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518897902223546:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.362364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:19.362412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:19.362506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:19.362671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518897650412328:2050] [node 3] ICS07 socket disconnect 430 reason# EndOfStream
2025-11-11T17:20:19.362695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518897650412328:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.362744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:19.362783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518897650412328:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:19.363604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:19.363832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518897685423433:2050] [node 3] ICS01 socket: 474 reason# EndOfStream
2025-11-11T17:20:19.363880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518893390456035:2050] [node 3] ICP30 unregister session Session# [4:7571518897685423433:2050] VirtualId# [4:7571518897685423424:0]
2025-11-11T17:20:19.363914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518897685423433:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.364023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:19.364069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:19.364399Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Couldn't connect to any resolved address
2025-11-11T17:20:19.364689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518893355444960:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:19.364737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518897650412328:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:19.364778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518893355444960:2050] [node 3] ICP30 unregister session Session# [6:7571518897650412328:2050] VirtualId# [6:7571518897650412314:0]
2025-11-11T17:20:19.364824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:19.364869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518893355444960:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.364885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15966) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:19.375341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518893355444960:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:20:19.469671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:19.469951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518897902223547:2098] [node 4] ICS01 socket: 439 reason# EndOfStream
2025-11-11T17:20:19.469996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518897902223508:2098] [node 4] ICP30 unregister session Session# [5:7571518897902223547:2098] VirtualId# [5:7571518897902223540:0]
2025-11-11T17:20:19.470030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518897902223547:2098] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.470125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:19.470170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:19.469681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:19.469950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518897650412357:2072] [node 4] ICS01 socket: 481 reason# EndOfStream
2025-11-11T17:20:19.469996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518897650412283:2072] [node 4] ICP30 unregister session Session# [6:7571518897650412357:2072] VirtualId# [6:7571518897650412355:0]
2025-11-11T17:20:19.470031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518897650412357:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.470125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:19.470169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:19.569921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:19.570231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518897650412353:2067] [node 5] ICS01 socket: 477 reason# EndOfStream
2025-11-11T17:20:19.570277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518897650412282:2067] [node 5] ICP30 unregister session Session# [6:7571518897650412353:2067] VirtualId# [6:7571518897650412346:0]
2025-11-11T17:20:19.570322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518897650412353:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:19.570452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:19.570513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:19.895401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518902598778882:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:19.895444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518902598778883:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:19.929666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518901705204374:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:19.929711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518901705204375:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:19.969919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518901986705955:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:19.969969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518901986705956:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:19.989360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518903438126030:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:19.989415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518903438126031:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:19.989452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518903438126032:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:20.008164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518904980986482:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:20.008209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518904980986483:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:20.008233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518904980986484:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:20.025221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518906330812727:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:20.025261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518906330812728:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:20.025288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518906330812729:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:20.437763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518907733093358:2071] [node 5] ICP01 ready to work
2025-11-11T17:20:20.431941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518906281673310:2102] [node 4] ICP01 ready to work
2025-11-11T17:20:20.438911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518906893746241:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:20.446032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518906281673315:2103] [node 5] ICP01 ready to work
2025-11-11T17:20:20.487418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518906893746242:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:20.548560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518901705204374:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(182123578ns) {Configure:104(182114537ns) {ProcessConfigured:135(76610ns) {StartInitialHandshake:157(70423ns) {DropHandshakes:469(3886ns) {DropIncomingHandshake:441(954ns) DropOutgoingHandshake:456(809ns)} GenerateSessionVirtualId:988(1059ns) PrepareNewSessionHandshake:476(2370ns) {SwitchToState:178(896ns)}}}}}
2025-11-11T17:20:20.524070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518906281673316:2104] [node 6] ICP01 ready to work
2025-11-11T17:20:20.522221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518907733093379:2075] [node 6] ICP01 ready to work
2025-11-11T17:20:20.549098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518906000171735:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:20.539348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518906893746243:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:20.550339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518906330812728:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(161501559ns) {Configure:104(161495862ns) {ProcessConfigured:135(75753640ns) {StartInitialHandshake:157(75750582ns) {DropHandshakes:469(3382ns) {DropIncomingHandshake:441(703ns) DropOutgoingHandshake:456(561ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:476(5836ns) {SwitchToState:178(2632ns)}}}}}
2025-11-11T17:20:20.558081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518906330812791:2060] [node 5] ICP01 ready to work
2025-11-11T17:20:20.558408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518906330812792:2063] [node 4] ICP01 ready to work
2025-11-11T17:20:20.567034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518902598778882:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17446525ns) {Configure:104(17441472ns) {ProcessConfigured:135(70414ns) {StartInitialHandshake:157(62093ns) {DropHandshakes:469(3241ns) {DropIncomingHandshake:441(618ns) DropOutgoingHandshake:456(618ns)} GenerateSessionVirtualId:988(1707ns) PrepareNewSessionHandshake:476(1968ns) {SwitchToState:178(800ns)}}}}}
2025-11-11T17:20:20.574588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518901986705956:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26989187ns) {Configure:104(26966647ns) {ProcessConfigured:135(39819ns) {StartInitialHandshake:157(38525ns) {DropHandshakes:469(939ns) {DropIncomingHandshake:441(280ns) DropOutgoingHandshake:456(229ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:476(759ns) {SwitchToState:178(198ns)}}}}}
2025-11-11T17:20:20.577879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518906000171736:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:20.585132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518906000171737:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:20.598424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518904980986483:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(193228679ns) {Configure:104(193215369ns) {ProcessConfigured:135(10862860ns) {StartInitialHandshake:157(10860159ns) {DropHandshakes:469(806ns) {DropIncomingHandshake:441(217ns) DropOutgoingHandshake:456(153ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:476(2282ns) {SwitchToState:178(1060ns)}}}}}
2025-11-11T17:20:20.597967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518903438126031:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(67830166ns) {Configure:104(67823257ns) {ProcessConfigured:135(67453ns) {StartInitialHandshake:157(64994ns) {DropHandshakes:469(5556ns) {DropIncomingHandshake:441(829ns) DropOutgoingHandshake:456(563ns)} GenerateSessionVirtualId:988(729ns) PrepareNewSessionHandshake:476(2162ns) {SwitchToState:178(624ns)}}}}}
2025-11-11T17:20:20.614933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518906000171736:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29088115ns) {Configure:104(29082241ns) {ProcessConfigured:135(56955ns) {StartInitialHandshake:157(54493ns) {DropHandshakes:469(2879ns) {DropIncomingHandshake:441(557ns) DropOutgoingHandshake:456(293ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:476(2123ns) {SwitchToState:178(787ns)}}}}}
2025-11-11T17:20:20.618542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518904980986547:2062] [node 6] ICP01 ready to work
2025-11-11T17:20:20.618917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518904980986548:2084] [node 4] ICP01 ready to work
2025-11-11T17:20:20.598231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518901986705955:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21131860ns) {Configure:104(21129785ns) {ProcessConfigured:135(265620ns) {StartInitialHandshake:157(262886ns) {DropHandshakes:469(2412ns) {DropIncomingHandshake:441(759ns) DropOutgoingHandshake:456(264ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:476(2651ns) {SwitchToState:178(981ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.626324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518906281673327:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5713
2025-11-11T17:20:20.646273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518903438126030:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33451311ns) {Configure:104(33446433ns) {ProcessConfigured:135(78091ns) {StartInitialHandshake:157(75700ns) {DropHandshakes:469(3948ns) {DropIncomingHandshake:441(1004ns) DropOutgoingHandshake:456(933ns)} GenerateSessionVirtualId:988(1012ns) PrepareNewSessionHandshake:476(2030ns) {SwitchToState:178(665ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.659715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518904980986548:2084] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33911848ns) {Configure:104(33907288ns) {ProcessConfigured:135(90626ns) {StartInitialHandshake:157(82556ns) {DropHandshakes:469(1758ns) {DropIncomingHandshake:441(410ns) DropOutgoingHandshake:456(257ns)} GenerateSessionVirtualId:988(796ns) PrepareNewSessionHandshake:476(2787ns) {SwitchToState:178(895ns)}}}}}
2025-11-11T17:20:20.660414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986549:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.661018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986558:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.676861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518907733093401:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32566
2025-11-11T17:20:20.677065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518907733093402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32566
2025-11-11T17:20:20.677132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518907733093403:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32566
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.710663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518906281673331:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5713
2025-11-11T17:20:20.710733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518906281673332:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5713
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.698332Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518907733093401:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.685916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518904980986549:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.725911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518906000171737:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(45934942ns) {Configure:104(45925783ns) {ProcessConfigured:135(4697429ns) {StartInitialHandshake:157(4695535ns) {DropHandshakes:469(2486ns) {DropIncomingHandshake:441(723ns) DropOutgoingHandshake:456(232ns)} GenerateSessionVirtualId:988(1732ns) PrepareNewSessionHandshake:476(4743ns) {SwitchToState:178(2739ns)}}}}}
2025-11-11T17:20:20.715332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986562:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.725671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518907733093403:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.753674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518906281673331:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.753865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518906281673332:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.754001Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518906281673322:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.754021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518906281673322:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.754552Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518906281673327:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.754991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518901986705955:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.761791Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518906281673322:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:20.750156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518904980986558:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.750325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518904980986484:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.774082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518904980986562:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.774608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986565:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.757949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746247:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.790813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518901705204375:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(64619366ns) {Configure:104(64611740ns) {ProcessConfigured:135(85929ns) {StartInitialHandshake:157(82831ns) {DropHandshakes:469(6898ns) {DropIncomingHandshake:441(944ns) DropOutgoingHandshake:456(608ns)} GenerateSessionVirtualId:988(1065ns) PrepareNewSessionHandshake:476(2045ns) {SwitchToState:178(679ns)}}}}}
2025-11-11T17:20:20.792283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518906000171738:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16477
2025-11-11T17:20:20.792792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518906000171747:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16477
2025-11-11T17:20:20.792895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518906000171748:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16477
2025-11-11T17:20:20.793021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518906000171749:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16477
2025-11-11T17:20:20.797646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518906000171750:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16477
2025-11-11T17:20:20.798047Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171738:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.757991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518906893746247:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.760237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518906893746257:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7980
2025-11-11T17:20:20.781873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518906893746247:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.793560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518906281673315:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.781957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518903438126032:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.804293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518907733093409:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32566
2025-11-11T17:20:20.804646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518904980986548:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.793654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:20.793704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518906281673315:2103] [node 5] ICP22 created new session: [3:7571518906281673338:2103]
2025-11-11T17:20:20.793762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518906281673338:2103] [node 5] ICS09 handshake done sender: [3:7571518906281673322:2112] self: [3:7571518906281673321:0] peer: [5:7571518904980986564:0] socket: 437 qp: -1
2025-11-11T17:20:20.793809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518906281673338:2103] [node 5] ICS10 traffic start
2025-11-11T17:20:20.793849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:20.802374Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518906281673320:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.802402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518906281673320:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.805600Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518907733093396:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.805627Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518907733093396:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.807076Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746257:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.807351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518902598778883:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.807404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:20.807459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518902598778883:2049] [node 3] ICP22 created new session: [1:7571518906893746260:2049]
2025-11-11T17:20:20.807579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518906893746260:2049] [node 3] ICS09 handshake done sender: [1:7571518906893746247:2111] self: [1:7571518906893746246:0] peer: [3:7571518906281673336:0] socket: 431 qp: -1
2025-11-11T17:20:20.807621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518906893746260:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:20.807668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:20.808553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518906893746259:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7980
2025-11-11T17:20:20.805342Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518904980986558:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:20.809575Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.810078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518906281673337:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5713
2025-11-11T17:20:20.810623Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518907733093396:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:20.811494Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518907733093402:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.811563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518907733093358:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.811622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:20.811668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518907733093358:2071] [node 5] ICP22 created new session: [4:7571518907733093411:2071]
2025-11-11T17:20:20.811716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518907733093411:2071] [node 5] ICS09 handshake done sender: [4:7571518907733093396:2112] self: [4:7571518907733093395:0] peer: [5:7571518904980986566:0] socket: 447 qp: -1
2025-11-11T17:20:20.811748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518907733093411:2071] [node 5] ICS10 traffic start
2025-11-11T17:20:20.811782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:20.812367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518903438126030:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.814055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518904980986484:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.814134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:20.814182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518904980986484:2050] [node 3] ICP22 created new session: [5:7571518904980986568:2050]
2025-11-11T17:20:20.814230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518904980986568:2050] [node 3] ICS09 handshake done sender: [5:7571518904980986558:2116] self: [5:7571518904980986564:0] peer: [3:7571518906281673321:0] socket: 441 qp: -1
2025-11-11T17:20:20.814263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518904980986568:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:20.814301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:20.814542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518904980986484:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.827042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746249:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.827084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518906893746249:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.829155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518906893746264:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7980
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.831442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518907733093415:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32566
2025-11-11T17:20:20.813156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171748:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.813252Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171750:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.813318Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171749:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.813383Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171747:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.832565Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518906281673327:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.832963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812805:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.830251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986567:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.830682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986570:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.833622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518901705204374:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.837881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518906281673320:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:20.838278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518901986705955:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.838346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:20.838402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518901986705955:2048] [node 1] ICP22 created new session: [3:7571518906281673342:2048]
2025-11-11T17:20:20.838448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518906281673342:2048] [node 1] ICS09 handshake done sender: [3:7571518906281673327:2115] self: [3:7571518906281673336:0] peer: [1:7571518906893746246:0] socket: 433 qp: -1
2025-11-11T17:20:20.838480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518906281673342:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:20.838517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:20.839059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518906281673310:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.839109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:20.839155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518906281673310:2102] [node 4] ICP22 created new session: [3:7571518906281673344:2102]
2025-11-11T17:20:20.839193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518906281673344:2102] [node 4] ICS09 handshake done sender: [3:7571518906281673320:2111] self: [3:7571518906281673319:0] peer: [4:7571518907733093408:0] socket: 436 qp: -1
2025-11-11T17:20:20.839223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518906281673344:2102] [node 4] ICS10 traffic start
2025-11-11T17:20:20.839249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.841818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986571:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.842295Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518904980986562:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.854206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812806:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.842498Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518906893746249:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:20.861090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518904980986548:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.861147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:20.862226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518904980986548:2084] [node 4] ICP22 created new session: [5:7571518904980986573:2084]
2025-11-11T17:20:20.862288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518904980986573:2084] [node 4] ICS09 handshake done sender: [5:7571518904980986562:2117] self: [5:7571518904980986566:0] peer: [4:7571518907733093395:0] socket: 448 qp: -1
2025-11-11T17:20:20.862320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518904980986573:2084] [node 4] ICS10 traffic start
2025-11-11T17:20:20.862388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:20.863041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518904980986567:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.863131Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518904980986571:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.878544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518904980986483:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.878653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518904980986482:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.879473Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:20.858681Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746245:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.880908Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518906330812805:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.881186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812807:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.881262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518906330812729:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.881799Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518906281673324:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.881834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518906281673324:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.858712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518906893746245:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.859631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518906893746241:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.859680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:20.859730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518906893746241:2107] [node 4] ICP22 created new session: [1:7571518906893746266:2107]
2025-11-11T17:20:20.859778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518906893746266:2107] [node 4] ICS09 handshake done sender: [1:7571518906893746249:2112] self: [1:7571518906893746248:0] peer: [4:7571518907733093414:0] socket: 432 qp: -1
2025-11-11T17:20:20.859813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518906893746266:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:20.859848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:20.874048Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746259:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.874900Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518906893746245:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:20.875256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518902598778882:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.875319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:20.875369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518902598778882:2048] [node 2] ICP22 created new session: [1:7571518906893746271:2048]
2025-11-11T17:20:20.875417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518906893746271:2048] [node 2] ICS09 handshake done sender: [1:7571518906893746245:2110] self: [1:7571518906893746244:0] peer: [2:7571518906000171758:0] socket: 430 qp: -1
2025-11-11T17:20:20.875448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518906893746271:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:20.875487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:20.880362Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:20.883637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746251:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.883668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518906893746251:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.886000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518901705204374:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518906000171747:2116] held: no
2025-11-11T17:20:20.886169Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171740:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.886189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518906000171740:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.886928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518906000171759:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16477
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.887594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.888098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986577:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.888368Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518906281673324:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:20.888624Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518906893746251:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:20.888925Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518907733093409:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.889302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518903438126032:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518907733093403:2117] held: no
2025-11-11T17:20:20.889506Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518907733093402:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:20.890209Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746264:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.890264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518906893746242:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.890319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:20.890364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518906893746242:2108] [node 5] ICP22 created new session: [1:7571518906893746274:2108]
2025-11-11T17:20:20.890404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518906893746274:2108] [node 5] ICS09 handshake done sender: [1:7571518906893746251:2113] self: [1:7571518906893746250:0] peer: [5:7571518904980986576:0] socket: 466 qp: -1
2025-11-11T17:20:20.890434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518906893746274:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:20.890479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:20.891518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518906281673316:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.891586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:20.891631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518906281673316:2104] [node 6] ICP22 created new session: [3:7571518906281673347:2104]
2025-11-11T17:20:20.891668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518906281673347:2104] [node 6] ICS09 handshake done sender: [3:7571518906281673324:2113] self: [3:7571518906281673323:0] peer: [6:7571518906330812808:0] socket: 473 qp: -1
2025-11-11T17:20:20.891697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518906281673347:2104] [node 6] ICS10 traffic start
2025-11-11T17:20:20.891756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:20.892490Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518906330812806:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.893240Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518904980986571:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:20.893718Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518906330812807:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.893862Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518906330812805:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:20.894152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518904980986482:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.894226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:20.894288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518904980986482:2048] [node 1] ICP22 created new session: [5:7571518904980986578:2048]
2025-11-11T17:20:20.894332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518904980986578:2048] [node 1] ICS09 handshake done sender: [5:7571518904980986571:2121] self: [5:7571518904980986576:0] peer: [1:7571518906893746250:0] socket: 470 qp: -1
2025-11-11T17:20:20.894368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518904980986578:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:20.894418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.896174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518907733093416:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32566
2025-11-11T17:20:20.896548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518906330812791:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.912084Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518906000171747:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:20.914403Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518906000171740:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:20.896625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518906330812727:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.896658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518906330812729:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.896732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:20.896775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518906330812729:2050] [node 3] ICP22 created new session: [6:7571518906330812812:2050]
2025-11-11T17:20:20.896823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518906330812812:2050] [node 3] ICS09 handshake done sender: [6:7571518906330812805:2115] self: [6:7571518906330812808:0] peer: [3:7571518906281673323:0] socket: 474 qp: -1
2025-11-11T17:20:20.896857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518906330812812:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:20.896886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.915578Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518904980986555:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.915615Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518904980986555:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.916776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518904980986580:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57141
2025-11-11T17:20:20.912296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518903438126030:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.912364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:20.912420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518903438126030:2048] [node 1] ICP22 created new session: [4:7571518907733093417:2048]
2025-11-11T17:20:20.912468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518907733093417:2048] [node 1] ICS09 handshake done sender: [4:7571518907733093402:2116] self: [4:7571518907733093414:0] peer: [1:7571518906893746248:0] socket: 439 qp: -1
2025-11-11T17:20:20.912506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518907733093417:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:20.912573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:20.913246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518903438126031:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.906599Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746253:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.906644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518906893746253:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.907148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518906893746241:2107] [node 4] ICP18 handshake (actor [1:7571518906893746264:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.919021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518906893746277:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7980
2025-11-11T17:20:20.918154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812814:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.918265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812816:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.919439Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518906893746253:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:20.919742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518906893746243:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.919791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:20.919838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518906893746243:2109] [node 6] ICP22 created new session: [1:7571518906893746278:2109]
2025-11-11T17:20:20.919884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518906893746278:2109] [node 6] ICS09 handshake done sender: [1:7571518906893746253:2114] self: [1:7571518906893746252:0] peer: [6:7571518906330812811:0] socket: 478 qp: -1
2025-11-11T17:20:20.919915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518906893746278:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:20.919950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:20.920845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518901705204374:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.920919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:20.920959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518901705204374:2048] [node 1] ICP22 created new session: [2:7571518906000171762:2048]
2025-11-11T17:20:20.921013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518906000171762:2048] [node 1] ICS09 handshake done sender: [2:7571518906000171747:2116] self: [2:7571518906000171758:0] peer: [1:7571518906893746244:0] socket: 456 qp: -1
2025-11-11T17:20:20.921046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518906000171762:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:20.921074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:20.921446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518906000171736:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.921519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:20.921562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518906000171736:2108] [node 5] ICP22 created new session: [2:7571518906000171764:2108]
2025-11-11T17:20:20.921642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518906000171764:2108] [node 5] ICS09 handshake done sender: [2:7571518906000171740:2112] self: [2:7571518906000171739:0] peer: [5:7571518904980986575:0] socket: 461 qp: -1
2025-11-11T17:20:20.921671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518906000171764:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:20.921698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.924451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812817:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.925350Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518904980986555:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:20.926318Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518907733093403:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:20.926704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518904980986547:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.926790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:20.928485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518903438126032:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.928561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:20.928624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518903438126032:2050] [node 3] ICP22 created new session: [4:7571518907733093421:2050]
2025-11-11T17:20:20.928678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518907733093421:2050] [node 3] ICS09 handshake done sender: [4:7571518907733093403:2117] self: [4:7571518907733093408:0] peer: [3:7571518906281673319:0] socket: 440 qp: -1
2025-11-11T17:20:20.928714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518907733093421:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:20.928749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:20.926837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518904980986547:2062] [node 6] ICP22 created new session: [5:7571518904980986582:2062]
2025-11-11T17:20:20.929659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518904980986582:2062] [node 6] ICS09 handshake done sender: [5:7571518904980986555:2114] self: [5:7571518904980986554:0] peer: [6:7571518906330812810:0] socket: 450 qp: -1
2025-11-11T17:20:20.929700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518904980986582:2062] [node 6] ICS10 traffic start
2025-11-11T17:20:20.929738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:20.930111Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518904980986567:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:20.930954Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171742:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.930994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518906000171742:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.938025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518906281673349:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5713
2025-11-11T17:20:20.938078Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:20.939569Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13791) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:20.940518Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518906893746277:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.940690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518906893746243:2109] [node 6] ICP18 handshake (actor [1:7571518906893746277:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.942946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518907733093423:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32566
2025-11-11T17:20:20.943341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518904980986483:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.943432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:20.943483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518904980986483:2049] [node 2] ICP22 created new session: [5:7571518904980986584:2049]
2025-11-11T17:20:20.943530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518904980986584:2049] [node 2] ICS09 handshake done sender: [5:7571518904980986567:2119] self: [5:7571518904980986575:0] peer: [2:7571518906000171739:0] socket: 464 qp: -1
2025-11-11T17:20:20.943585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518904980986584:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:20.943622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:20.943902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518904980986483:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:20.945073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518906330812791:2060] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13791) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518906330812806:2116] held: no
2025-11-11T17:20:20.945352Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518906330812807:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:20.945755Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:20.946575Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518906000171742:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:20.947319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518906000171735:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.947414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:20.947457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518906000171735:2107] [node 4] ICP22 created new session: [2:7571518906000171768:2107]
2025-11-11T17:20:20.947504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518906000171768:2107] [node 4] ICS09 handshake done sender: [2:7571518906000171742:2113] self: [2:7571518906000171741:0] peer: [4:7571518907733093420:0] socket: 462 qp: -1
2025-11-11T17:20:20.947560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518906000171768:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:20.947594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:20.949934Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518907733093409:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:20.950232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518903438126031:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.950287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:20.950332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518903438126031:2049] [node 2] ICP22 created new session: [4:7571518907733093424:2049]
2025-11-11T17:20:20.950378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518907733093424:2049] [node 2] ICS09 handshake done sender: [4:7571518907733093409:2118] self: [4:7571518907733093420:0] peer: [2:7571518906000171741:0] socket: 463 qp: -1
2025-11-11T17:20:20.950420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518907733093424:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:20.950479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:20.952153Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518906330812806:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.952864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518906330812727:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.952948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:20.952993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518906330812727:2048] [node 1] ICP22 created new session: [6:7571518906330812819:2048]
2025-11-11T17:20:20.953047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518906330812819:2048] [node 1] ICS09 handshake done sender: [6:7571518906330812807:2117] self: [6:7571518906330812811:0] peer: [1:7571518906893746252:0] socket: 479 qp: -1
2025-11-11T17:20:20.953083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518906330812819:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:20.953115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:20.953344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518906330812727:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:20.953767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518906330812814:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.953808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518906330812791:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.953879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:20.953925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518906330812791:2060] [node 5] ICP22 created new session: [6:7571518906330812821:2060]
2025-11-11T17:20:20.953970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518906330812821:2060] [node 5] ICS09 handshake done sender: [6:7571518906330812806:2116] self: [6:7571518906330812810:0] peer: [5:7571518904980986554:0] socket: 476 qp: -1
2025-11-11T17:20:20.953998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518906330812821:2060] [node 5] ICS10 traffic start
2025-11-11T17:20:20.954023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:20.954357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812818:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.954723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518906330812728:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.962633Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171744:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.962675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518906000171744:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.963778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518906330812818:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.963916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518906330812792:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.965807Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518906281673349:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.965947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518901986705956:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:20.966164Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518907733093400:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.966200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518907733093400:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.967493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812825:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.967574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518906330812826:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30739
2025-11-11T17:20:20.967874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518907733093400:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:20.968457Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518906330812818:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:20.969815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518906000171744:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:20.970278Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518906000171746:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:20.970307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518906000171746:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:20.970833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518906000171737:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.968708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518906330812792:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.968758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:20.968805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518906330812792:2063] [node 4] ICP22 created new session: [6:7571518906330812827:2063]
2025-11-11T17:20:20.968852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518906330812827:2063] [node 4] ICS09 handshake done sender: [6:7571518906330812818:2122] self: [6:7571518906330812824:0] peer: [4:7571518907733093399:0] socket: 434 qp: -1
2025-11-11T17:20:20.968885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518906330812827:2063] [node 4] ICS10 traffic start
2025-11-11T17:20:20.968923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:20.971351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:20.971407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518906000171737:2109] [node 6] ICP22 created new session: [2:7571518906000171771:2109]
2025-11-11T17:20:20.971451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518906000171771:2109] [node 6] ICS09 handshake done sender: [2:7571518906000171744:2114] self: [2:7571518906000171743:0] peer: [6:7571518906330812823:0] socket: 451 qp: -1
2025-11-11T17:20:20.971484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518906000171771:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:20.971519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:20.972560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518907733093379:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.972617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:20.972659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518907733093379:2075] [node 6] ICP22 created new session: [4:7571518907733093427:2075]
2025-11-11T17:20:20.972697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518907733093427:2075] [node 6] ICS09 handshake done sender: [4:7571518907733093400:2114] self: [4:7571518907733093399:0] peer: [6:7571518906330812824:0] socket: 467 qp: -1
2025-11-11T17:20:20.972731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518907733093427:2075] [node 6] ICS10 traffic start
2025-11-11T17:20:20.972765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:20.974504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518906281673351:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5713
2025-11-11T17:20:20.975699Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:20.977088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518906330812728:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518906330812814:2119] held: no
2025-11-11T17:20:20.977247Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518906330812814:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:20.977801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518906330812728:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.977881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:20.977934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518906330812728:2049] [node 2] ICP22 created new session: [6:7571518906330812829:2049]
2025-11-11T17:20:20.977978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518906330812829:2049] [node 2] ICS09 handshake done sender: [6:7571518906330812814:2119] self: [6:7571518906330812823:0] peer: [2:7571518906000171743:0] socket: 482 qp: -1
2025-11-11T17:20:20.978014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518906330812829:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:20.978053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:20.978643Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518906000171746:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:20.979387Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518906281673349:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:20.979841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518901986705956:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:20.979900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:20.979944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518901986705956:2049] [node 2] ICP22 created new session: [3:7571518906281673352:2049]
2025-11-11T17:20:20.979985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518906281673352:2049] [node 2] ICS09 handshake done sender: [3:7571518906281673349:2119] self: [3:7571518906281673350:0] peer: [2:7571518906000171745:0] socket: 490 qp: -1
2025-11-11T17:20:20.980017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518906281673352:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:20.980049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:20.981797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518901705204375:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:20.981864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:20.981911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518901705204375:2049] [node 3] ICP22 created new session: [2:7571518906000171774:2049]
2025-11-11T17:20:20.981954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518906000171774:2049] [node 3] ICS09 handshake done sender: [2:7571518906000171746:2115] self: [2:7571518906000171745:0] peer: [3:7571518906281673350:0] socket: 458 qp: -1
2025-11-11T17:20:20.981986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518906000171774:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:20.982013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:21.029154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:21.029382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518906000171762:2048] [node 1] ICS01 socket: 456 reason# EndOfStream
2025-11-11T17:20:21.029421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518901705204374:2048] [node 1] ICP30 unregister session Session# [2:7571518906000171762:2048] VirtualId# [2:7571518906000171758:0]
2025-11-11T17:20:21.029458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518906000171762:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.029539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:21.029577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:21.033670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:21.033930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518906281673342:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-11-11T17:20:21.033982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518901986705955:2048] [node 1] ICP30 unregister session Session# [3:7571518906281673342:2048] VirtualId# [3:7571518906281673336:0]
2025-11-11T17:20:21.034053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518906281673342:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.034144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:21.034190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:21.037770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:21.038034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518906330812819:2048] [node 1] ICS01 socket: 479 reason# EndOfStream
2025-11-11T17:20:21.038071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518906330812727:2048] [node 1] ICP30 unregister session Session# [6:7571518906330812819:2048] VirtualId# [6:7571518906330812811:0]
2025-11-11T17:20:21.038105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518906330812819:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.038185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:21.038235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:21.037788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:21.038034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518907733093417:2048] [node 1] ICS01 socket: 439 reason# EndOfStream
2025-11-11T17:20:21.038079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518903438126030:2048] [node 1] ICP30 unregister session Session# [4:7571518907733093417:2048] VirtualId# [4:7571518907733093414:0]
2025-11-11T17:20:21.038105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518907733093417:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.038185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:21.038234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:21.038322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:21.038502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518904980986578:2048] [node 1] ICS01 socket: 470 reason# EndOfStream
2025-11-11T17:20:21.038539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518904980986482:2048] [node 1] ICP30 unregister session Session# [5:7571518904980986578:2048] VirtualId# [5:7571518904980986576:0]
2025-11-11T17:20:21.038565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518904980986578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.038626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:21.038664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:21.080488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:21.080718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518906281673352:2049] [node 2] ICS01 socket: 490 reason# EndOfStream
2025-11-11T17:20:21.080762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518901986705956:2049] [node 2] ICP30 unregister session Session# [3:7571518906281673352:2049] VirtualId# [3:7571518906281673350:0]
2025-11-11T17:20:21.080797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518906281673352:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.080899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:21.080942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:21.093795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:21.094033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518907733093424:2049] [node 2] ICS01 socket: 463 reason# EndOfStream
2025-11-11T17:20:21.094078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518903438126031:2049] [node 2] ICP30 unregister session Session# [4:7571518907733093424:2049] VirtualId# [4:7571518907733093420:0]
2025-11-11T17:20:21.094115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518907733093424:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.094201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:21.094256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:21.094377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:21.094548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518906330812829:2049] [node 2] ICS01 socket: 482 reason# EndOfStream
2025-11-11T17:20:21.094585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518906330812728:2049] [node 2] ICP30 unregister session Session# [6:7571518906330812829:2049] VirtualId# [6:7571518906330812823:0]
2025-11-11T17:20:21.094613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518906330812829:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.094669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:21.094705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:21.101751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:21.102029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518904980986584:2049] [node 2] ICS01 socket: 464 reason# EndOfStream
2025-11-11T17:20:21.102078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518904980986483:2049] [node 2] ICP30 unregister session Session# [5:7571518904980986584:2049] VirtualId# [5:7571518904980986575:0]
2025-11-11T17:20:21.102122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518904980986584:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.102272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:21.102335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:21.309875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:21.313730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:21.310125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518907733093421:2050] [node 3] ICS01 socket: 440 reason# EndOfStream
2025-11-11T17:20:21.313993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518904980986568:2050] [node 3] ICS01 socket: 441 reason# EndOfStream
2025-11-11T17:20:21.310168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518903438126032:2050] [node 3] ICP30 unregister session Session# [4:7571518907733093421:2050] VirtualId# [4:7571518907733093408:0]
2025-11-11T17:20:21.314037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518904980986484:2050] [node 3] ICP30 unregister session Session# [5:7571518904980986568:2050] VirtualId# [5:7571518904980986564:0]
2025-11-11T17:20:21.310207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518907733093421:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.314074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518904980986568:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.310308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:21.314182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:21.310361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:21.314240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:21.313751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:21.313991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518906330812812:2050] [node 3] ICS01 socket: 474 reason# EndOfStream
2025-11-11T17:20:21.314035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518906330812729:2050] [node 3] ICP30 unregister session Session# [6:7571518906330812812:2050] VirtualId# [6:7571518906330812808:0]
2025-11-11T17:20:21.314069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518906330812812:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.314178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:21.314229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:21.373770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:21.374030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518904980986573:2084] [node 4] ICS01 socket: 448 reason# EndOfStream
2025-11-11T17:20:21.374074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518904980986548:2084] [node 4] ICP30 unregister session Session# [5:7571518904980986573:2084] VirtualId# [5:7571518904980986566:0]
2025-11-11T17:20:21.374115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518904980986573:2084] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.374202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:21.374254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:21.375603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:21.375804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518906330812827:2063] [node 4] ICS01 socket: 434 reason# EndOfStream
2025-11-11T17:20:21.375844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518906330812792:2063] [node 4] ICP30 unregister session Session# [6:7571518906330812827:2063] VirtualId# [6:7571518906330812824:0]
2025-11-11T17:20:21.375874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518906330812827:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.375953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:21.375999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:21.425795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:21.426046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518906330812821:2060] [node 5] ICS01 socket: 476 reason# EndOfStream
2025-11-11T17:20:21.426091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518906330812791:2060] [node 5] ICP30 unregister session Session# [6:7571518906330812821:2060] VirtualId# [6:7571518906330812810:0]
2025-11-11T17:20:21.426132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518906330812821:2060] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:21.426273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:21.426327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:21.621936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518911984783345:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:21.621992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518911984783346:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:21.680578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518909678921222:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:21.680627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518909678921223:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:21.710139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518910445337081:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:21.710185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518910445337082:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:21.752620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518910483449757:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:21.752674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518910483449758:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:21.752707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518910483449759:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:21.774930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518910759484977:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:21.774973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518910759484978:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:21.775001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518910759484979:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:21.831514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518910861618428:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:21.831573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518910861618429:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:21.831606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518910861618430:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:22.223336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518914740304439:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:22.223681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518914740304440:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:22.293658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518915156585784:2103] [node 5] ICP01 ready to work
2025-11-11T17:20:22.305926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518913973888581:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:22.306248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518913973888582:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:22.306527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518913973888583:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:22.322353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518916279750705:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:22.335034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518916279750706:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:22.347524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518910759484979:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(105623530ns) {Configure:104(105617775ns) {ProcessConfigured:135(80844ns) {StartInitialHandshake:157(77547ns) {DropHandshakes:469(2581ns) {DropIncomingHandshake:441(707ns) DropOutgoingHandshake:456(573ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:476(2145ns) {SwitchToState:178(718ns)}}}}}
2025-11-11T17:20:22.348039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518915054452338:2109] [node 4] ICP01 ready to work
2025-11-11T17:20:22.373843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518915156585791:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:22.369704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518914778417117:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.354022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518916279750707:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:22.425945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518914778417118:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:22.415850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518910861618430:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(74596795ns) {Configure:104(74593737ns) {ProcessConfigured:135(64402ns) {StartInitialHandshake:157(61499ns) {DropHandshakes:469(2449ns) {DropIncomingHandshake:441(642ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(1126ns) PrepareNewSessionHandshake:476(1441ns) {SwitchToState:178(300ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.421825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518915054452346:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.433670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518914740304440:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(140326827ns) {Configure:104(140322163ns) {ProcessConfigured:135(20690155ns) {StartInitialHandshake:157(20686821ns) {DropHandshakes:469(2335ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(558ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:476(5491ns) {SwitchToState:178(2828ns)}}}}}
2025-11-11T17:20:22.422060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452343:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.439292Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518915054452346:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16712362ns) {Configure:104(16707718ns) {ProcessConfigured:135(73040ns) {StartInitialHandshake:157(69814ns) {DropHandshakes:469(4352ns) {DropIncomingHandshake:441(890ns) DropOutgoingHandshake:456(540ns)} GenerateSessionVirtualId:988(1108ns) PrepareNewSessionHandshake:476(2976ns) {SwitchToState:178(1129ns)}}}}}
2025-11-11T17:20:22.439723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518915054452343:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.440092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452349:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.440263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518910759484978:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.474942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888589:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.437634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518910445337081:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(177152382ns) {Configure:104(177146354ns) {ProcessConfigured:135(35215218ns) {StartInitialHandshake:157(35211747ns) {DropHandshakes:469(2204ns) {DropIncomingHandshake:441(679ns) DropOutgoingHandshake:456(434ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:476(5560ns) {SwitchToState:178(2982ns)}}}}}
2025-11-11T17:20:22.498116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452354:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.502041Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518915054452343:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:22.510953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518910483449758:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(81247111ns) {Configure:104(81241067ns) {ProcessConfigured:135(47621ns) {StartInitialHandshake:157(45645ns) {DropHandshakes:469(3400ns) {DropIncomingHandshake:441(768ns) DropOutgoingHandshake:456(617ns)} GenerateSessionVirtualId:988(861ns) PrepareNewSessionHandshake:476(908ns) {SwitchToState:178(268ns)}}}}}
2025-11-11T17:20:22.474984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518913973888589:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.541519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585801:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.466018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518914740304449:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.554289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518914740304457:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48255
2025-11-11T17:20:22.531566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518910759484978:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.554496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518914740304458:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48255
2025-11-11T17:20:22.547883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452355:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.548010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452357:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.475894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518913973888595:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37882
2025-11-11T17:20:22.553776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:22.553845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518910759484978:2049] [node 2] ICP22 created new session: [5:7571518915054452356:2049]
2025-11-11T17:20:22.496564Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518913973888589:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:22.506010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518909678921223:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(288495977ns) {Configure:104(288485827ns) {ProcessConfigured:135(8422536ns) {StartInitialHandshake:157(7009241ns) {DropHandshakes:469(4519ns) {DropIncomingHandshake:441(1472ns) DropOutgoingHandshake:456(1017ns)} GenerateSessionVirtualId:988(12419ns) PrepareNewSessionHandshake:476(4383ns) {SwitchToState:178(1955ns)}}}}}
2025-11-11T17:20:22.506215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888595:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.537416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518911984783346:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(195007670ns) {Configure:104(195001908ns) {ProcessConfigured:135(86847ns) {StartInitialHandshake:157(83517ns) {DropHandshakes:469(3901ns) {DropIncomingHandshake:441(1247ns) DropOutgoingHandshake:456(700ns)} GenerateSessionVirtualId:988(818ns) PrepareNewSessionHandshake:476(2380ns) {SwitchToState:178(834ns)}}}}}
2025-11-11T17:20:22.558090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585802:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.602989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518914778417117:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(91767015ns) {Configure:104(91761850ns) {ProcessConfigured:135(15243189ns) {StartInitialHandshake:157(15239600ns) {DropHandshakes:469(4267ns) {DropIncomingHandshake:441(1051ns) DropOutgoingHandshake:456(833ns)} GenerateSessionVirtualId:988(1266ns) PrepareNewSessionHandshake:476(5658ns) {SwitchToState:178(2616ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.610861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518910861618429:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(156128003ns) {Configure:104(156120789ns) {ProcessConfigured:135(27030278ns) {StartInitialHandshake:157(27027670ns) {DropHandshakes:469(2485ns) {DropIncomingHandshake:441(355ns) DropOutgoingHandshake:456(702ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:476(3177ns) {SwitchToState:178(651ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.612443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585804:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.612921Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518915156585801:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.613073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518915156585804:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.613147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518910861618430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.613338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518910861618429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.590312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518913973888582:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.558344Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518915054452349:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.594032Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304457:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.560170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518915054452356:2049] [node 2] ICS09 handshake done sender: [5:7571518915054452343:2113] self: [5:7571518915054452353:0] peer: [2:7571518913973888588:0] socket: 435 qp: -1
2025-11-11T17:20:22.560218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518915054452356:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:22.560280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:22.560743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518910759484978:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(34569594ns) {HandleHandshakeStatus:329(29018229ns) {DropIncomingHandshake:441(1385ns) DropOutgoingHandshake:456(10628420ns) SwitchToState:178(1628ns) ProcessPendingSessionEvents:495(108306ns) {ForwardSessionEventToSession:612(60911ns) ForwardSessionEventToSession:612(3649ns) ForwardSessionEventToSession:612(6782ns) ForwardSessionEventToSession:612(5091ns) ForwardSessionEventToSession:612(3195ns) ForwardSessionEventToSession:612(5681ns) ForwardSessionEventToSession:612(3343ns) ForwardSessionEventToSession:612(2794ns) ForwardSessionEventToSession:612(5041ns) ForwardSessionEventToSession:612(2754ns)}}}
2025-11-11T17:20:22.540135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518916279750708:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62446
2025-11-11T17:20:22.641842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518915156585802:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.642033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518910861618428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.644310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585808:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.645524Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518915156585802:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.653758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518910861618428:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.653838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:22.653884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518910861618428:2048] [node 1] ICP22 created new session: [6:7571518915156585809:2048]
2025-11-11T17:20:22.653941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518915156585809:2048] [node 1] ICS09 handshake done sender: [6:7571518915156585802:2115] self: [6:7571518915156585807:0] peer: [1:7571518916279750713:0] socket: 461 qp: -1
2025-11-11T17:20:22.653980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518915156585809:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:22.654016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:22.590407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:22.540522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518916279750719:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62446
2025-11-11T17:20:22.604085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417121:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.590462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518913973888582:2108] [node 5] ICP22 created new session: [2:7571518913973888600:2108]
2025-11-11T17:20:22.678082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518915054452355:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.604617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417128:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
2025-11-11T17:20:22.594180Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304458:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.540634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518916279750720:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62446
2025-11-11T17:20:22.590512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518913973888600:2108] [node 5] ICS09 handshake done sender: [2:7571518913973888589:2112] self: [2:7571518913973888588:0] peer: [5:7571518915054452353:0] socket: 434 qp: -1
2025-11-11T17:20:22.590559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518913973888600:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:22.590601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:22.605555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518913973888603:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37882
2025-11-11T17:20:22.625972Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888587:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.626006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518913973888587:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.638146Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518913973888587:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:22.638200Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888591:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.638239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518913973888591:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.679459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518913973888581:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.679509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:22.679568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518913973888581:2107] [node 4] ICP22 created new session: [2:7571518913973888606:2107]
2025-11-11T17:20:22.679641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518913973888606:2107] [node 4] ICS09 handshake done sender: [2:7571518913973888587:2111] self: [2:7571518913973888586:0] peer: [4:7571518914778417133:0] socket: 433 qp: -1
2025-11-11T17:20:22.679679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518913973888606:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:22.679721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:22.684737Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518913973888591:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:22.684376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.611805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518914740304459:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48255
2025-11-11T17:20:22.678661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452361:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.709929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518913973888583:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.710011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:22.710073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518913973888583:2109] [node 6] ICP22 created new session: [2:7571518913973888609:2109]
2025-11-11T17:20:22.710148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518913973888609:2109] [node 6] ICS09 handshake done sender: [2:7571518913973888591:2113] self: [2:7571518913973888590:0] peer: [6:7571518915156585806:0] socket: 445 qp: -1
2025-11-11T17:20:22.710200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518913973888609:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:22.710252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.684494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.604714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417129:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
2025-11-11T17:20:22.678834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518910759484979:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.540734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518916279750721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62446
2025-11-11T17:20:22.715839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888603:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.560045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518916279750707:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(116878813ns) {Configure:104(116874116ns) {ProcessConfigured:135(27544959ns) {StartInitialHandshake:157(17676174ns) {DropHandshakes:469(4431ns) {DropIncomingHandshake:441(861ns) DropOutgoingHandshake:456(571ns)} GenerateSessionVirtualId:988(3154ns) PrepareNewSessionHandshake:476(4598ns) {SwitchToState:178(1711ns)}}}}}
2025-11-11T17:20:22.655682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304459:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.582440Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750720:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.605882Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750719:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.622128Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750708:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.642931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750714:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.642957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518916279750714:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.644634Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518916279750714:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:22.658663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750721:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.658713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518916279750707:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.658793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:22.658855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518916279750707:2109] [node 6] ICP22 created new session: [1:7571518916279750728:2109]
2025-11-11T17:20:22.677710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518916279750728:2109] [node 6] ICS09 handshake done sender: [1:7571518916279750714:2114] self: [1:7571518916279750713:0] peer: [6:7571518915156585807:0] socket: 460 qp: -1
2025-11-11T17:20:22.685806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518916279750728:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:22.693843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:22.694393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518916279750707:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(35436401ns) {HandleHandshakeStatus:329(35433554ns) {DropIncomingHandshake:441(14334ns) DropOutgoingHandshake:456(857ns) SwitchToState:178(1212ns) ProcessPendingSessionEvents:495(118845ns) {ForwardSessionEventToSession:612(71392ns) ForwardSessionEventToSession:612(3692ns) ForwardSessionEventToSession:612(6762ns) ForwardSessionEventToSession:612(2141ns) ForwardSessionEventToSession:612(3147ns) ForwardSessionEventToSession:612(5013ns) ForwardSessionEventToSession:612(5870ns) ForwardSessionEventToSession:612(6350ns) ForwardSessionEventToSession:612(2826ns) ForwardSessionEventToSession:612(2424ns)}}}
2025-11-11T17:20:22.697765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518911984783345:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(17410413ns) {IncomingHandshake:261(5532596ns) {DropIncomingHandshake:441(784ns) PrepareNewSessionHandshake:476(5497816ns) {SwitchToState:178(2006ns)} GenerateSessionVirtualId:988(395ns) IssueIncomingHandshakeReply:193(5729ns)}}
2025-11-11T17:20:22.706988Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750710:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.707035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518916279750710:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.731453Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750712:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.731481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518916279750712:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.735520Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518916279750710:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:22.736367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518916279750712:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:22.737248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518916279750705:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.737324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:22.737375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518916279750705:2108] [node 5] ICP22 created new session: [1:7571518916279750735:2108]
2025-11-11T17:20:22.737423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518916279750735:2108] [node 5] ICS09 handshake done sender: [1:7571518916279750710:2112] self: [1:7571518916279750709:0] peer: [5:7571518915054452364:0] socket: 458 qp: -1
2025-11-11T17:20:22.737469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518916279750735:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:22.737512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:22.737704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518916279750706:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.737777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:22.738047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518916279750706:2107] [node 4] ICP22 created new session: [1:7571518916279750737:2107]
2025-11-11T17:20:22.618002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518914778417121:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.715490Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518915156585804:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:22.658963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304447:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.659005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518914740304447:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.682311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304451:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.682339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518914740304451:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.683408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518914740304463:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48255
2025-11-11T17:20:22.690872Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518914740304451:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:22.690962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518914740304447:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:22.685953Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518915054452357:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.715992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518913973888581:2107] [node 4] ICP18 handshake (actor [2:7571518913973888603:2116]) is too old
2025-11-11T17:20:22.618316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518910483449758:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.650636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518914778417128:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.650835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518914778417129:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.651427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417135:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
2025-11-11T17:20:22.709421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518910483449757:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(237715753ns) {Configure:104(237708969ns) {ProcessConfigured:135(21060002ns) {StartInitialHandshake:157(21055687ns) {DropHandshakes:469(3512ns) {DropIncomingHandshake:441(780ns) DropOutgoingHandshake:456(680ns)} GenerateSessionVirtualId:988(1137ns) PrepareNewSessionHandshake:476(5471ns) {SwitchToState:178(1966ns)}}}}}
2025-11-11T17:20:22.714108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518910483449757:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.714954Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518914778417121:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:22.726071Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:22.726957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518910483449758:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.727015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:22.727071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518910483449758:2049] [node 2] ICP22 created new session: [4:7571518914778417140:2049]
2025-11-11T17:20:22.727117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518914778417140:2049] [node 2] ICS09 handshake done sender: [4:7571518914778417121:2112] self: [4:7571518914778417133:0] peer: [2:7571518913973888586:0] socket: 441 qp: -1
2025-11-11T17:20:22.727150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518914778417140:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:22.727203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:22.727457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518910483449758:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:22.727874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417139:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
2025-11-11T17:20:22.735165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417143:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
2025-11-11T17:20:22.736950Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518914778417128:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:22.737862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518910483449757:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.737919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:22.757656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518910483449757:2048] [node 1] ICP22 created new session: [4:7571518914778417144:2048]
2025-11-11T17:20:22.773861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518914778417144:2048] [node 1] ICS09 handshake done sender: [4:7571518914778417128:2116] self: [4:7571518914778417138:0] peer: [1:7571518916279750711:0] socket: 464 qp: -1
2025-11-11T17:20:22.773918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518914778417144:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.782234Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518914778417139:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.783418Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518914778417127:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.783454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518914778417127:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.773999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518916279750737:2107] [node 4] ICS09 handshake done sender: [1:7571518916279750712:2113] self: [1:7571518916279750711:0] peer: [4:7571518914778417138:0] socket: 459 qp: -1
2025-11-11T17:20:22.784786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518916279750737:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:22.784881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.790620Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518914778417127:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:22.686326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518910759484977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.686410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452363:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.725743Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518915054452355:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:22.758091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518910759484979:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.758175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:22.758226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518910759484979:2050] [node 3] ICP22 created new session: [5:7571518915054452366:2050]
2025-11-11T17:20:22.758268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518915054452366:2050] [node 3] ICS09 handshake done sender: [5:7571518915054452355:2118] self: [5:7571518915054452362:0] peer: [3:7571518914740304450:0] socket: 451 qp: -1
2025-11-11T17:20:22.758305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518915054452366:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:22.758341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:22.773718Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518915054452361:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.781857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452365:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
2025-11-11T17:20:22.793907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518915054452338:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.716481Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518915156585801:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:22.802184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518916279750705:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(33167316ns) {HandleHandshakeStatus:329(33163826ns) {DropIncomingHandshake:441(17222ns) DropOutgoingHandshake:456(852ns) SwitchToState:178(1027ns) ProcessPendingSessionEvents:495(32729508ns) {ForwardSessionEventToSession:612(61078ns) ForwardSessionEventToSession:612(3855ns) ForwardSessionEventToSession:612(7124251ns) ForwardSessionEventToSession:612(5846ns) ForwardSessionEventToSession:612(3111ns) ForwardSessionEventToSession:612(14984195ns) ForwardSessionEventToSession:612(8258ns) ForwardSessionEventToSession:612(6061782ns) ForwardSessionEventToSession:612(30886ns) ForwardSessionEventToSession:612(3484ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.802863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518914740304440:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.802948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:22.803005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518914740304440:2108] [node 6] ICP22 created new session: [3:7571518914740304467:2108]
2025-11-11T17:20:22.803092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518914740304467:2108] [node 6] ICS09 handshake done sender: [3:7571518914740304447:2112] self: [3:7571518914740304445:0] peer: [6:7571518915156585805:0] socket: 448 qp: -1
2025-11-11T17:20:22.803138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518914740304467:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:22.803176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:22.803775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518914740304449:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.803817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:22.803863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518914740304449:2110] [node 5] ICP22 created new session: [3:7571518914740304469:2110]
2025-11-11T17:20:22.803900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518914740304469:2110] [node 5] ICS09 handshake done sender: [3:7571518914740304451:2114] self: [3:7571518914740304450:0] peer: [5:7571518915054452362:0] socket: 450 qp: -1
2025-11-11T17:20:22.803929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518914740304469:2110] [node 5] ICS10 traffic start
2025-11-11T17:20:22.803954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:22.806409Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304463:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.806553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518914740304466:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48255
2025-11-11T17:20:22.806971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518910445337082:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.799664Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518915054452357:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:22.835622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518916279750706:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(97660281ns) {HandleHandshakeStatus:329(97652485ns) {DropIncomingHandshake:441(262ns) DropOutgoingHandshake:456(368ns) SwitchToState:178(321ns) ProcessPendingSessionEvents:495(21578488ns) {ForwardSessionEventToSession:612(21513012ns) ForwardSessionEventToSession:612(5012ns) ForwardSessionEventToSession:612(9670ns) ForwardSessionEventToSession:612(3480ns) ForwardSessionEventToSession:612(2772ns) ForwardSessionEventToSession:612(5915ns) ForwardSessionEventToSession:612(3021ns) ForwardSessionEventToSession:612(8266ns) ForwardSessionEventToSession:612(2485ns) ForwardSessionEventToSession:612(1973ns)}}}
2025-11-11T17:20:22.834140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518910759484977:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.834211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:22.793475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518914778417117:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.716976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518910861618429:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.793543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:22.798291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518914778417117:2108] [node 6] ICP22 created new session: [4:7571518914778417147:2108]
2025-11-11T17:20:22.799549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:22.807476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518914778417147:2108] [node 6] ICS09 handshake done sender: [4:7571518914778417127:2115] self: [4:7571518914778417126:0] peer: [6:7571518915156585819:0] socket: 468 qp: -1
2025-11-11T17:20:22.817872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518914778417147:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:22.817933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.812473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518913973888611:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37882
2025-11-11T17:20:22.834270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518910759484977:2048] [node 1] ICP22 created new session: [5:7571518915054452370:2048]
2025-11-11T17:20:22.834337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518915054452370:2048] [node 1] ICS09 handshake done sender: [5:7571518915054452357:2119] self: [5:7571518915054452364:0] peer: [1:7571518916279750709:0] socket: 462 qp: -1
2025-11-11T17:20:22.834377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518915054452370:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:22.834418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:22.868717Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:22.869095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518910759484977:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.717056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:22.853743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518914740304474:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48255
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.878198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518914778417117:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(84498737ns) {HandleHandshakeStatus:329(84496500ns) {DropIncomingHandshake:441(15748ns) DropOutgoingHandshake:456(741ns) SwitchToState:178(506ns) ProcessPendingSessionEvents:495(7280351ns) {ForwardSessionEventToSession:612(7207625ns) ForwardSessionEventToSession:612(23101ns) ForwardSessionEventToSession:612(3341ns) ForwardSessionEventToSession:612(8057ns) ForwardSessionEventToSession:612(3145ns) ForwardSessionEventToSession:612(2763ns) ForwardSessionEventToSession:612(5238ns) ForwardSessionEventToSession:612(2968ns) ForwardSessionEventToSession:612(5733ns) ForwardSessionEventToSession:612(5535ns)}}}
2025-11-11T17:20:22.879029Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518914778417123:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.879051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518914778417123:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.880308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417150:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
2025-11-11T17:20:22.880587Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518914778417123:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:22.812586Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888598:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.882123Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518914740304463:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:22.881087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518914778417118:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.812613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518913973888598:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.845351Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518913973888598:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:22.852460Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888611:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.852790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518913973888613:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37882
2025-11-11T17:20:22.874568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518909678921223:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.874641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:22.874685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518909678921223:2049] [node 3] ICP22 created new session: [2:7571518913973888615:2049]
2025-11-11T17:20:22.874755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518913973888615:2049] [node 3] ICS09 handshake done sender: [2:7571518913973888598:2115] self: [2:7571518913973888597:0] peer: [3:7571518914740304473:0] socket: 439 qp: -1
2025-11-11T17:20:22.874796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518913973888615:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:22.874836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:22.887887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518913973888613:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.717111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518910861618429:2049] [node 2] ICP22 created new session: [6:7571518915156585814:2049]
2025-11-11T17:20:22.894050Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518915054452351:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.894100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518915054452351:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.899066Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:22.881157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:22.881202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518914778417118:2109] [node 5] ICP22 created new session: [4:7571518914778417152:2109]
2025-11-11T17:20:22.881247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518914778417152:2109] [node 5] ICS09 handshake done sender: [4:7571518914778417123:2113] self: [4:7571518914778417122:0] peer: [5:7571518915054452369:0] socket: 467 qp: -1
2025-11-11T17:20:22.881280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518914778417152:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:22.881311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:22.901147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518910483449757:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(95946959ns) {HandleHandshakeStatus:329(95944492ns) {DropIncomingHandshake:441(336ns) DropOutgoingHandshake:456(4898ns) SwitchToState:178(173ns) ProcessPendingSessionEvents:495(14051516ns) {ForwardSessionEventToSession:612(13987823ns) ForwardSessionEventToSession:612(14226ns) ForwardSessionEventToSession:612(5058ns) ForwardSessionEventToSession:612(5464ns) ForwardSessionEventToSession:612(3286ns) ForwardSessionEventToSession:612(2723ns) ForwardSessionEventToSession:612(7610ns) ForwardSessionEventToSession:612(3177ns) ForwardSessionEventToSession:612(2967ns) ForwardSessionEventToSession:612(5857ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.717157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518915156585814:2049] [node 2] ICS09 handshake done sender: [6:7571518915156585804:2117] self: [6:7571518915156585806:0] peer: [2:7571518913973888590:0] socket: 463 qp: -1
2025-11-11T17:20:22.913851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518909678921222:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.906179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518915054452338:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518915054452361:2120] held: no
2025-11-11T17:20:22.717197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518915156585814:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:22.914292Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:22.915801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518915054452372:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3672
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.942227Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518915054452351:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:22.942484Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518915054452361:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:22.922147Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750718:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.923414Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518914778417150:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.946022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518913973888618:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37882
2025-11-11T17:20:22.717255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:22.933802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518910483449759:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.918035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518910445337082:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.918096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:22.959404Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:22.922192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518916279750718:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.942807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518915054452338:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.961741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518909678921222:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518913973888613:2118] held: no
2025-11-11T17:20:22.961905Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518913973888613:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:22.717767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518910861618430:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.923072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518910445337082:2049] [node 2] ICP22 created new session: [3:7571518914740304475:2049]
2025-11-11T17:20:22.942862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:22.717818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:22.945886Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518916279750718:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:22.942913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518915054452338:2109] [node 4] ICP22 created new session: [5:7571518915054452374:2109]
2025-11-11T17:20:22.942980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518915054452374:2109] [node 4] ICS09 handshake done sender: [5:7571518915054452361:2120] self: [5:7571518915054452369:0] peer: [4:7571518914778417122:0] socket: 471 qp: -1
2025-11-11T17:20:22.946690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518915054452374:2109] [node 4] ICS10 traffic start
2025-11-11T17:20:22.950017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:22.954003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518915054452346:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.954068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:22.954116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518915054452346:2108] [node 6] ICP22 created new session: [5:7571518915054452376:2108]
2025-11-11T17:20:22.954163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518915054452376:2108] [node 6] ICS09 handshake done sender: [5:7571518915054452351:2116] self: [5:7571518915054452350:0] peer: [6:7571518915156585824:0] socket: 482 qp: -1
2025-11-11T17:20:22.954206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518915054452376:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:22.954247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:22.949827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518914740304475:2049] [node 2] ICS09 handshake done sender: [3:7571518914740304463:2119] self: [3:7571518914740304473:0] peer: [2:7571518913973888597:0] socket: 477 qp: -1
2025-11-11T17:20:22.978624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518909678921222:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.978688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:22.978744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518909678921222:2048] [node 1] ICP22 created new session: [2:7571518913973888619:2048]
2025-11-11T17:20:22.978793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518913973888619:2048] [node 1] ICS09 handshake done sender: [2:7571518913973888613:2118] self: [2:7571518913973888617:0] peer: [1:7571518916279750717:0] socket: 493 qp: -1
2025-11-11T17:20:22.978825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518913973888619:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:22.978865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:22.974703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518915054452338:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(23278640ns) {HandleHandshakeStatus:329(23276086ns) {DropIncomingHandshake:441(1050ns) DropOutgoingHandshake:456(583ns) SwitchToState:178(781ns) ProcessPendingSessionEvents:495(710ns)}}
2025-11-11T17:20:22.717872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518910861618430:2050] [node 3] ICP22 created new session: [6:7571518915156585816:2050]
2025-11-11T17:20:22.971640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518914778417155:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6671
2025-11-11T17:20:22.946118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518911984783345:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.717913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518915156585816:2050] [node 3] ICS09 handshake done sender: [6:7571518915156585801:2114] self: [6:7571518915156585805:0] peer: [3:7571518914740304445:0] socket: 453 qp: -1
2025-11-11T17:20:22.733045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518915156585816:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:22.733096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:22.781116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518910861618430:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(63138470ns) {HandleHandshakeStatus:329(63134709ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(5988ns) SwitchToState:178(327ns) ProcessPendingSessionEvents:495(7110174ns) {ForwardSessionEventToSession:612(7054965ns) ForwardSessionEventToSession:612(11896ns) ForwardSessionEventToSession:612(3379ns) ForwardSessionEventToSession:612(5890ns) ForwardSessionEventToSession:612(3166ns) ForwardSessionEventToSession:612(4456ns) ForwardSessionEventToSession:612(5748ns) ForwardSessionEventToSession:612(2144ns) ForwardSessionEventToSession:612(2573ns) ForwardSessionEventToSession:612(5315ns)}}}
2025-11-11T17:20:22.781466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585818:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.782912Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518915156585818:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.783034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518915156585791:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.790314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585820:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.791318Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518915156585818:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:22.791580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518915156585791:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.791668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:22.791721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518915156585791:2108] [node 4] ICP22 created new session: [6:7571518915156585821:2108]
2025-11-11T17:20:22.791794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518915156585821:2108] [node 4] ICS09 handshake done sender: [6:7571518915156585818:2121] self: [6:7571518915156585819:0] peer: [4:7571518914778417126:0] socket: 440 qp: -1
2025-11-11T17:20:22.791845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518915156585821:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:22.791891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:22.874182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585823:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.878215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518915156585823:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.878375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518915156585784:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.941929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518915156585825:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-11T17:20:22.945747Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518915156585823:2123] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:22.958047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518915156585784:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.960636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:22.966779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518915156585784:2103] [node 5] ICP22 created new session: [6:7571518915156585826:2103]
2025-11-11T17:20:22.977816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518915156585826:2103] [node 5] ICS09 handshake done sender: [6:7571518915156585823:2123] self: [6:7571518915156585824:0] peer: [5:7571518915054452350:0] socket: 490 qp: -1
2025-11-11T17:20:22.977870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518915156585826:2103] [node 5] ICS10 traffic start
2025-11-11T17:20:22.985702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:22.986275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518915156585784:2103] [node 5] ICP35 event processing took too much time PendingConnection:278(28012815ns) {HandleHandshakeStatus:329(28009514ns) {DropIncomingHandshake:441(1029ns) DropOutgoingHandshake:456(6012ns) SwitchToState:178(927ns) ProcessPendingSessionEvents:495(126441ns) {ForwardSessionEventToSession:612(70147ns) ForwardSessionEventToSession:612(6746ns) ForwardSessionEventToSession:612(3840ns) ForwardSessionEventToSession:612(5951ns) ForwardSessionEventToSession:612(3596ns) ForwardSessionEventToSession:612(3233ns) ForwardSessionEventToSession:612(11256ns) ForwardSessionEventToSession:612(2742ns) ForwardSessionEventToSession:612(2620ns) ForwardSessionEventToSession:612(6361ns)}}}
2025-11-11T17:20:22.954170Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304466:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.978382Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518914778417150:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:22.979794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518910483449759:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:22.979837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:22.979880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518910483449759:2050] [node 3] ICP22 created new session: [4:7571518914778417156:2050]
2025-11-11T17:20:22.979948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518914778417156:2050] [node 3] ICS09 handshake done sender: [4:7571518914778417150:2122] self: [4:7571518914778417154:0] peer: [3:7571518914740304441:0] socket: 455 qp: -1
2025-11-11T17:20:22.979982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518914778417156:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:22.980013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:22.954823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518914740304475:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:22.954077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:22.965792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:22.966159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518914740304442:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.966195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518914740304442:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.970244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518910445337081:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:22.972969Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518914740304442:2109] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:22.998077Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:22.999485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518910445337082:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(48985481ns) {HandleHandshakeStatus:329(48981298ns) {DropIncomingHandshake:441(861ns) DropOutgoingHandshake:456(6040ns) SwitchToState:178(988ns) ProcessPendingSessionEvents:495(97063ns) {ForwardSessionEventToSession:612(52519ns) ForwardSessionEventToSession:612(5557ns) ForwardSessionEventToSession:612(5259ns) ForwardSessionEventToSession:612(3375ns) ForwardSessionEventToSession:612(4917ns) ForwardSessionEventToSession:612(3020ns) ForwardSessionEventToSession:612(3559ns) ForwardSessionEventToSession:612(5283ns) ForwardSessionEventToSession:612(2124ns) ForwardSessionEventToSession:612(5253ns)}}}
2025-11-11T17:20:22.999567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518914740304439:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.999629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:23.001970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518914740304439:2107] [node 4] ICP22 created new session: [3:7571518914740304480:2107]
2025-11-11T17:20:23.002030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518914740304480:2107] [node 4] ICS09 handshake done sender: [3:7571518914740304442:2109] self: [3:7571518914740304441:0] peer: [4:7571518914778417154:0] socket: 452 qp: -1
2025-11-11T17:20:23.002076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518914740304480:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:23.002176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:23.003913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518910445337081:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518914740304466:2120] held: no
2025-11-11T17:20:23.006878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518910445337082:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:22.954131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518911984783345:2048] [node 2] ICP22 created new session: [1:7571518916279750741:2048]
2025-11-11T17:20:22.954200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518916279750741:2048] [node 2] ICS09 handshake done sender: [1:7571518916279750718:2116] self: [1:7571518916279750717:0] peer: [2:7571518913973888617:0] socket: 491 qp: -1
2025-11-11T17:20:22.954237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518916279750741:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:22.954300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:22.971887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518916279750716:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:22.971911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518916279750716:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:22.972975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518916279750716:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:22.973277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518911984783346:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:22.973335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:22.973385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518911984783346:2049] [node 3] ICP22 created new session: [1:7571518916279750744:2049]
2025-11-11T17:20:22.973427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518916279750744:2049] [node 3] ICS09 handshake done sender: [1:7571518916279750716:2115] self: [1:7571518916279750715:0] peer: [3:7571518914740304478:0] socket: 484 qp: -1
2025-11-11T17:20:22.973460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518916279750744:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:22.973504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:23.003179Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518911984783345:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(24682761ns) {HandleHandshakeStatus:329(24679685ns) {DropIncomingHandshake:441(25967ns) DropOutgoingHandshake:456(582ns) SwitchToState:178(1335ns) ProcessPendingSessionEvents:495(16320450ns) {ForwardSessionEventToSession:612(3543345ns) ForwardSessionEventToSession:612(7269ns) ForwardSessionEventToSession:612(4028ns) ForwardSessionEventToSession:612(2772038ns) ForwardSessionEventToSession:612(4037ns) ForwardSessionEventToSession:612(5537577ns) ForwardSessionEventToSession:612(2505951ns) ForwardSessionEventToSession:612(3657ns) ForwardSessionEventToSession:612(1917804ns) ForwardSessionEventToSession:612(3745ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:23.014178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518914740304479:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48255
2025-11-11T17:20:23.020339Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518914740304466:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:23.023493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518910445337081:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:23.023620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:23.023691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518910445337081:2048] [node 1] ICP22 created new session: [3:7571518919035271778:2048]
2025-11-11T17:20:23.023777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518919035271778:2048] [node 1] ICS09 handshake done sender: [3:7571518914740304466:2120] self: [3:7571518914740304478:0] peer: [1:7571518916279750715:0] socket: 489 qp: -1
2025-11-11T17:20:23.023830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518919035271778:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:23.023876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:23.071014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:23.071244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518913973888619:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-11T17:20:23.071298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518909678921222:2048] [node 1] ICP30 unregister session Session# [2:7571518913973888619:2048] VirtualId# [2:7571518913973888617:0]
2025-11-11T17:20:23.071334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518913973888619:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.071433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:23.071476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:23.072902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:23.073089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518919035271778:2048] [node 1] ICS01 socket: 489 reason# EndOfStream
2025-11-11T17:20:23.073129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518910445337081:2048] [node 1] ICP30 unregister session Session# [3:7571518919035271778:2048] VirtualId# [3:7571518914740304478:0]
2025-11-11T17:20:23.073163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518919035271778:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.073235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:23.073274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:23.077748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:23.077979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518915054452370:2048] [node 1] ICS01 socket: 462 reason# EndOfStream
2025-11-11T17:20:23.078021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518910759484977:2048] [node 1] ICP30 unregister session Session# [5:7571518915054452370:2048] VirtualId# [5:7571518915054452364:0]
2025-11-11T17:20:23.078053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518915054452370:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.078147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:23.078202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:23.077807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:23.077999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518914778417144:2048] [node 1] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-11T17:20:23.078028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518914778417144:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.078127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:23.078183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518914778417144:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:23.078726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:23.078916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518915156585809:2048] [node 1] ICS01 socket: 461 reason# EndOfStream
2025-11-11T17:20:23.078949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518910861618428:2048] [node 1] ICP30 unregister session Session# [6:7571518915156585809:2048] VirtualId# [6:7571518915156585807:0]
2025-11-11T17:20:23.078984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518915156585809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.079057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:23.079095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:23.079969Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Couldn't connect to any resolved address
2025-11-11T17:20:23.080229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518910483449757:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:23.080277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518914778417144:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:23.080311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518910483449757:2048] [node 1] ICP30 unregister session Session# [4:7571518914778417144:2048] VirtualId# [4:7571518914778417138:0]
2025-11-11T17:20:23.080355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:23.080383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518910483449757:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:23.080398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:23.089581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518910483449757:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:23.143933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:23.144199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518914740304475:2049] [node 2] ICS07 socket disconnect 477 reason# EndOfStream
2025-11-11T17:20:23.144226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518914740304475:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.144315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:23.144356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518914740304475:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:23.148396Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:20:23.148553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518910445337082:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:20:23.148594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518914740304475:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:23.149732Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Couldn't connect to any resolved address
2025-11-11T17:20:23.149821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518910445337082:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:23.149881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518914740304475:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:23.150025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518910445337082:2049] [node 2] ICP30 unregister session Session# [3:7571518914740304475:2049] VirtualId# [3:7571518914740304473:0]
2025-11-11T17:20:23.150085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:23.150148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518910445337082:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:23.150226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:23.157340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:23.157610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518915054452356:2049] [node 2] ICS01 socket: 435 reason# EndOfStream
2025-11-11T17:20:23.157658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518910759484978:2049] [node 2] ICP30 unregister session Session# [5:7571518915054452356:2049] VirtualId# [5:7571518915054452353:0]
2025-11-11T17:20:23.157693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518915054452356:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.157783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:23.157832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:23.158202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:23.157757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:23.160988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518915156585814:2049] [node 2] ICS01 socket: 463 reason# EndOfStream
2025-11-11T17:20:23.157943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518914778417140:2049] [node 2] ICS01 socket: 441 reason# EndOfStream
2025-11-11T17:20:23.157977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518910483449758:2049] [node 2] ICP30 unregister session Session# [4:7571518914778417140:2049] VirtualId# [4:7571518914778417133:0]
2025-11-11T17:20:23.158003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518914778417140:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.158094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:23.158142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:23.161342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518910861618429:2049] [node 2] ICP30 unregister session Session# [6:7571518915156585814:2049] VirtualId# [6:7571518915156585806:0]
2025-11-11T17:20:23.161385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518915156585814:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.161484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:23.161532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:23.161676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518910445337082:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:23.249846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:23.249835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:23.250082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518915054452366:2050] [node 3] ICS01 socket: 451 reason# EndOfStream
2025-11-11T17:20:23.250081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518915156585816:2050] [node 3] ICS01 socket: 453 reason# EndOfStream
2025-11-11T17:20:23.250118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518910759484979:2050] [node 3] ICP30 unregister session Session# [5:7571518915054452366:2050] VirtualId# [5:7571518915054452362:0]
2025-11-11T17:20:23.250121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518910861618430:2050] [node 3] ICP30 unregister session Session# [6:7571518915156585816:2050] VirtualId# [6:7571518915156585805:0]
2025-11-11T17:20:23.250152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518915054452366:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.250153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518915156585816:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.250265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:23.250342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:23.250329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:23.250385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:23.250507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:23.250723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518914778417156:2050] [node 3] ICS01 socket: 455 reason# EndOfStream
2025-11-11T17:20:23.250757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518910483449759:2050] [node 3] ICP30 unregister session Session# [4:7571518914778417156:2050] VirtualId# [4:7571518914778417154:0]
2025-11-11T17:20:23.250787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518914778417156:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.250891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:23.250949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:23.305764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:23.306005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518915054452374:2109] [node 4] ICS01 socket: 471 reason# EndOfStream
2025-11-11T17:20:23.306044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518915054452338:2109] [node 4] ICP30 unregister session Session# [5:7571518915054452374:2109] VirtualId# [5:7571518915054452369:0]
2025-11-11T17:20:23.306077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518915054452374:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.306167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:23.306212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:23.306317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:23.306521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518915156585821:2108] [node 4] ICS01 socket: 440 reason# EndOfStream
2025-11-11T17:20:23.306556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518915156585791:2108] [node 4] ICP30 unregister session Session# [6:7571518915156585821:2108] VirtualId# [6:7571518915156585819:0]
2025-11-11T17:20:23.306613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518915156585821:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.306695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:23.306736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:23.394039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:23.394451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518915156585826:2103] [node 5] ICS07 socket disconnect 490 reason# EndOfStream
2025-11-11T17:20:23.394518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518915156585826:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:23.394748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:23.394834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518915156585826:2103] [node 5] ICS15 start handshake
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:23.527108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518920914590133:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:23.527164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518920914590134:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:23.548449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518921463528319:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:23.548526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518921463528320:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:23.568101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518918552865660:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:23.568110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518918552865661:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:23.605573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518921048792993:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:23.605633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518921048792994:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:23.605660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518921048792995:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:23.626485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518919940681153:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:23.626529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518919940681154:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:23.626563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518919940681155:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:23.678608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518921260843516:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:23.678663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518921260843518:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:23.678678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518921260843517:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:24.313694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518925209557492:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:24.314127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518925209557494:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:24.302121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518925555810872:2106] [node 5] ICP01 ready to work
2025-11-11T17:20:24.398060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518925555810872:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(95588252ns) {Configure:104(95583049ns) {ProcessConfigured:135(47812849ns) {StartInitialHandshake:157(47808619ns) {DropHandshakes:469(2597ns) {DropIncomingHandshake:441(653ns) DropOutgoingHandshake:456(514ns)} GenerateSessionVirtualId:988(1325ns) PrepareNewSessionHandshake:476(5268ns) {SwitchToState:178(2734ns)}}}}}
2025-11-11T17:20:24.347856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518925209557492:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33438750ns) {Configure:104(33432712ns) {ProcessConfigured:135(113013ns) {StartInitialHandshake:157(109258ns) {DropHandshakes:469(4324ns) {DropIncomingHandshake:441(836ns) DropOutgoingHandshake:456(620ns)} GenerateSessionVirtualId:988(1069ns) PrepareNewSessionHandshake:476(3183ns) {SwitchToState:178(1597ns)}}}}}
2025-11-11T17:20:24.446023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518925758495680:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:24.421856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518922847833018:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:24.376890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518925209557499:2110] [node 6] ICP01 ready to work
2025-11-11T17:20:24.490672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518925758495679:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:24.466588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518922847833021:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:24.490465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518924235648511:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:24.490692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518925343760351:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.565928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518925555810879:2110] [node 4] ICP01 ready to work
2025-11-11T17:20:24.594248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518921260843518:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27976954ns) {Configure:104(27973261ns) {ProcessConfigured:135(82682ns) {StartInitialHandshake:157(79337ns) {DropHandshakes:469(2619ns) {DropIncomingHandshake:441(563ns) DropOutgoingHandshake:456(577ns)} GenerateSessionVirtualId:988(1404ns) PrepareNewSessionHandshake:476(1465ns) {SwitchToState:178(571ns)}}}}}
2025-11-11T17:20:24.596376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518925555810882:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.591376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518925758495681:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:24.565757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518922847833022:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:24.603291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518925343760353:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:24.617480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518925209557501:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59183
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.646001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518921463528319:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(161913266ns) {Configure:104(161902456ns) {ProcessConfigured:135(63871469ns) {StartInitialHandshake:157(44980012ns) {DropHandshakes:469(5659ns) {DropIncomingHandshake:441(793ns) DropOutgoingHandshake:456(1057ns)} GenerateSessionVirtualId:988(2946ns) PrepareNewSessionHandshake:476(3192ns) {SwitchToState:178(893ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.667083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518921048792993:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(290257149ns) {Configure:104(290250635ns) {ProcessConfigured:135(78109ns) {StartInitialHandshake:157(75684ns) {DropHandshakes:469(2716ns) {DropIncomingHandshake:441(968ns) DropOutgoingHandshake:456(428ns)} GenerateSessionVirtualId:988(1311ns) PrepareNewSessionHandshake:476(1514ns) {SwitchToState:178(597ns)}}}}}
2025-11-11T17:20:24.670189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518922847833032:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-11-11T17:20:24.622550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518925209557494:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(284768550ns) {Configure:104(284762461ns) {ProcessConfigured:135(84887ns) {StartInitialHandshake:157(82303ns) {DropHandshakes:469(4010ns) {DropIncomingHandshake:441(1031ns) DropOutgoingHandshake:456(582ns)} GenerateSessionVirtualId:988(1182ns) PrepareNewSessionHandshake:476(2551ns) {SwitchToState:178(773ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.656197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518925758495680:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(57570381ns) {Configure:104(57562792ns) {ProcessConfigured:135(72435ns) {StartInitialHandshake:157(70128ns) {DropHandshakes:469(5117ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(450ns)} GenerateSessionVirtualId:988(1415ns) PrepareNewSessionHandshake:476(1899ns) {SwitchToState:178(667ns)}}}}}
2025-11-11T17:20:24.678331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518925758495681:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21868711ns) {Configure:104(21863424ns) {ProcessConfigured:135(127547ns) {StartInitialHandshake:157(88244ns) {DropHandshakes:469(3435ns) {DropIncomingHandshake:441(759ns) DropOutgoingHandshake:456(752ns)} GenerateSessionVirtualId:988(1221ns) PrepareNewSessionHandshake:476(2228ns) {SwitchToState:178(838ns)}}}}}
2025-11-11T17:20:24.702847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518925555810887:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:24.717842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518920914590134:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(289819029ns) {Configure:104(289813928ns) {ProcessConfigured:135(29613845ns) {StartInitialHandshake:157(24041562ns) {DropHandshakes:469(3544ns) {DropIncomingHandshake:441(582ns) DropOutgoingHandshake:456(425ns)} GenerateSessionVirtualId:988(3553ns) PrepareNewSessionHandshake:476(7253ns) {SwitchToState:178(3389ns)}}}}}
2025-11-11T17:20:24.723809Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557501:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.680761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518925758495688:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6487
2025-11-11T17:20:24.709677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518925758495688:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.719743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518921463528319:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.698668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518925343760351:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16701180ns) {Configure:104(16697544ns) {ProcessConfigured:135(84797ns) {StartInitialHandshake:157(81118ns) {DropHandshakes:469(2361ns) {DropIncomingHandshake:441(440ns) DropOutgoingHandshake:456(519ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:476(1898ns) {SwitchToState:178(745ns)}}}}}
2025-11-11T17:20:24.738240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557490:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.738295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518925209557490:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.723165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518918552865661:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(138718060ns) {Configure:104(138710410ns) {ProcessConfigured:135(77954ns) {StartInitialHandshake:157(75067ns) {DropHandshakes:469(3690ns) {DropIncomingHandshake:441(969ns) DropOutgoingHandshake:456(784ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:476(2056ns) {SwitchToState:178(844ns)}}}}}
2025-11-11T17:20:24.702991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518925555810888:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:24.754995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518921463528320:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(164168140ns) {Configure:104(156781463ns) {ProcessConfigured:135(20127465ns) {StartInitialHandshake:157(20123728ns) {DropHandshakes:469(2585ns) {DropIncomingHandshake:441(575ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(949ns) PrepareNewSessionHandshake:476(3454ns) {SwitchToState:178(567ns)}}}}}
2025-11-11T17:20:24.731563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518919940681155:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(261586332ns) {Configure:104(261579044ns) {ProcessConfigured:135(80482ns) {StartInitialHandshake:157(77717ns) {DropHandshakes:469(3318ns) {DropIncomingHandshake:441(694ns) DropOutgoingHandshake:456(759ns)} GenerateSessionVirtualId:988(900ns) PrepareNewSessionHandshake:476(4512ns) {SwitchToState:178(698ns)}}}}}
2025-11-11T17:20:24.742960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518919940681154:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(252199187ns) {Configure:104(252192758ns) {ProcessConfigured:135(53684ns) {StartInitialHandshake:157(51759ns) {DropHandshakes:469(1976ns) {DropIncomingHandshake:441(326ns) DropOutgoingHandshake:456(382ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:476(790ns) {SwitchToState:178(171ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.740387Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518922847833032:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.737792Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518925555810887:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.770483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518919940681153:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(303597014ns) {Configure:104(303590104ns) {ProcessConfigured:135(70624ns) {StartInitialHandshake:157(67945ns) {DropHandshakes:469(3421ns) {DropIncomingHandshake:441(1159ns) DropOutgoingHandshake:456(566ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:476(1999ns) {SwitchToState:178(682ns)}}}}}
2025-11-11T17:20:24.771221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648519:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.771390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648520:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.771498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648521:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.782101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518924235648522:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:24.787591Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518924235648519:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.797807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518921048792994:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(360411128ns) {Configure:104(360405270ns) {ProcessConfigured:135(17207695ns) {StartInitialHandshake:157(17206084ns) {DropHandshakes:469(3572ns) {DropIncomingHandshake:441(893ns) DropOutgoingHandshake:456(360ns)} GenerateSessionVirtualId:988(1403ns) PrepareNewSessionHandshake:476(3426ns) {SwitchToState:178(196ns)}}}}}
2025-11-11T17:20:24.798857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518925343760356:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
2025-11-11T17:20:24.799026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518925343760357:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
2025-11-11T17:20:24.799240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518925343760364:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
2025-11-11T17:20:24.739374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518921260843517:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(148200298ns) {Configure:104(148185960ns) {ProcessConfigured:135(70893ns) {StartInitialHandshake:157(69051ns) {DropHandshakes:469(1727ns) {DropIncomingHandshake:441(266ns) DropOutgoingHandshake:456(439ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:476(2293ns) {SwitchToState:178(729ns)}}}}}
2025-11-11T17:20:24.750696Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518918552865660:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(392547666ns) {Configure:104(392542534ns) {ProcessConfigured:135(12733783ns) {StartInitialHandshake:157(12730158ns) {DropHandshakes:469(2735ns) {DropIncomingHandshake:441(680ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(1044ns) PrepareNewSessionHandshake:476(4511ns) {SwitchToState:178(2240ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.778509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518925758495697:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6487
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.769693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557503:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.805240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518924235648520:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.821045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518919940681155:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.837709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518924235648521:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.846517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518919940681154:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.817671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518925343760356:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.834206Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518925343760364:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.848737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518921048792994:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.854300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518921048792995:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(207475462ns) {Configure:104(207466701ns) {ProcessConfigured:135(11962882ns) {StartInitialHandshake:157(11960205ns) {DropHandshakes:469(4509ns) {DropIncomingHandshake:441(1127ns) DropOutgoingHandshake:456(736ns)} GenerateSessionVirtualId:988(1175ns) PrepareNewSessionHandshake:476(3560ns) {SwitchToState:178(1220ns)}}}}}
2025-11-11T17:20:24.854498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518921048792995:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.769763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518925209557503:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.782543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518922847833038:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-11-11T17:20:24.739679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518921260843516:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.863186Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518925758495685:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.850975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648528:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.772287Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518925555810882:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.830999Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518922847833026:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.869700Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518925343760357:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.786709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518925209557511:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59183
2025-11-11T17:20:24.831026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518922847833026:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.868086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648531:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.863224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518925758495685:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.779041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518925555810888:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.795977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518925209557490:2107] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:24.905744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518921048792993:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.835254Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518922847833026:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:24.874073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518925758495690:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.814291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518921260843518:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.868153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648532:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.802307Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518925209557503:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.853656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518922847833028:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.874124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518925758495690:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.829981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518925555810879:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(223768653ns) {Configure:104(223755589ns) {ProcessConfigured:135(15933278ns) {StartInitialHandshake:157(15928913ns) {DropHandshakes:469(3474ns) {DropIncomingHandshake:441(761ns) DropOutgoingHandshake:456(786ns)} GenerateSessionVirtualId:988(1014ns) PrepareNewSessionHandshake:476(10110ns) {SwitchToState:178(1764ns)}}}}}
2025-11-11T17:20:24.870111Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518924235648528:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.924753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518921048792993:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(35942489ns) {IncomingHandshake:261(35939194ns) {DropIncomingHandshake:441(784ns) PrepareNewSessionHandshake:476(5219ns) {SwitchToState:178(3480ns)} GenerateSessionVirtualId:988(1101ns) IssueIncomingHandshakeReply:193(31533807ns) {SwitchToState:178(2951ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.858983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518920914590133:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.925018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518925343760361:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.877758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518925758495687:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.830095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518921260843517:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.874355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518924235648511:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.853712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518922847833028:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.859036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.925054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518925343760361:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.859091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518920914590133:2048] [node 2] ICP22 created new session: [1:7571518925209557514:2048]
2025-11-11T17:20:24.859137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518925209557514:2048] [node 2] ICS09 handshake done sender: [1:7571518925209557490:2107] self: [1:7571518925209557489:0] peer: [2:7571518925758495694:0] socket: 442 qp: -1
2025-11-11T17:20:24.859174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518925209557514:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:24.859212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:24.859714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518925209557499:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:24.859763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:24.859809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518925209557499:2110] [node 6] ICP22 created new session: [1:7571518925209557516:2110]
2025-11-11T17:20:24.859847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518925209557516:2110] [node 6] ICS09 handshake done sender: [1:7571518925209557503:2113] self: [1:7571518925209557502:0] peer: [6:7571518925555810891:0] socket: 448 qp: -1
2025-11-11T17:20:24.859877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518925209557516:2110] [node 6] ICS10 traffic start
2025-11-11T17:20:24.859904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:24.862236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518925209557513:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59183
2025-11-11T17:20:24.870148Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557511:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.896817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557497:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.896849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518925209557497:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.906264Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557513:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.933804Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518925209557497:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:24.953986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518925209557523:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59183
2025-11-11T17:20:24.974737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518925209557492:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:24.974821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:24.998035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518925209557492:2108] [node 4] ICP22 created new session: [1:7571518925209557524:2108]
2025-11-11T17:20:24.830755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518925555810893:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:24.877796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518925758495687:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.906109Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:24.854804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518922847833038:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.001686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557523:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.018800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518929504524821:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59183
2025-11-11T17:20:25.027518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518925209557524:2108] [node 4] ICS09 handshake done sender: [1:7571518925209557497:2111] self: [1:7571518925209557496:0] peer: [4:7571518925343760372:0] socket: 447 qp: -1
2025-11-11T17:20:25.030064Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557507:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.030103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518925209557507:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.934923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518925343760374:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.886349Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518925758495697:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.838309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518925555810897:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:25.030168Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:20:25.040857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518925209557524:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:25.052653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:25.062054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518925209557508:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.062086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518925209557508:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.874442Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518922847833024:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.062890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518925209557507:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:25.072323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518925209557494:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.072403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:25.072458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518925209557494:2109] [node 5] ICP22 created new session: [1:7571518929504524826:2109]
2025-11-11T17:20:25.078481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518929504524826:2109] [node 5] ICS09 handshake done sender: [1:7571518925209557507:2114] self: [1:7571518925209557506:0] peer: [5:7571518924235648542:0] socket: 481 qp: -1
2025-11-11T17:20:25.083165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518929504524826:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:25.086582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:24.838377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518925555810898:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:24.958024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518919940681155:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518924235648520:2114] held: no
2025-11-11T17:20:24.874470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518922847833024:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.882335Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518922847833028:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:24.882686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518922847833022:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:24.882778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:24.882834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518922847833022:2109] [node 6] ICP22 created new session: [3:7571518922847833042:2109]
2025-11-11T17:20:24.882887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518922847833042:2109] [node 6] ICS09 handshake done sender: [3:7571518922847833026:2112] self: [3:7571518922847833025:0] peer: [6:7571518925555810892:0] socket: 440 qp: -1
2025-11-11T17:20:24.882922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518922847833042:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:24.882982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:24.883486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518922847833021:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:24.883550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:24.883587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518922847833021:2108] [node 5] ICP22 created new session: [3:7571518922847833044:2108]
2025-11-11T17:20:24.883628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518922847833044:2108] [node 5] ICS09 handshake done sender: [3:7571518922847833028:2113] self: [3:7571518922847833027:0] peer: [5:7571518924235648529:0] socket: 441 qp: -1
2025-11-11T17:20:25.021645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518925343760373:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
2025-11-11T17:20:24.958580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518924235648520:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:24.891474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518925758495699:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6487
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.883655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518922847833044:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:25.087495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518925209557492:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(120639883ns) {HandleHandshakeStatus:329(120635909ns) {DropIncomingHandshake:441(26884ns) DropOutgoingHandshake:456(437ns) SwitchToState:178(522ns) ProcessPendingSessionEvents:495(4494971ns) {ForwardSessionEventToSession:612(4444387ns) ForwardSessionEventToSession:612(8455ns) ForwardSessionEventToSession:612(3365ns) ForwardSessionEventToSession:612(2850ns) ForwardSessionEventToSession:612(9261ns) ForwardSessionEventToSession:612(3134ns) ForwardSessionEventToSession:612(2963ns) ForwardSessionEventToSession:612(5322ns) ForwardSessionEventToSession:612(2468ns) ForwardSessionEventToSession:612(3829ns)}}}
2025-11-11T17:20:24.869876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518921260843518:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(38216716ns) {IncomingHandshake:261(38214340ns) {DropIncomingHandshake:441(454ns) PrepareNewSessionHandshake:476(2010ns) {SwitchToState:178(868ns)} GenerateSessionVirtualId:988(732ns) IssueIncomingHandshakeReply:193(31660096ns) {SwitchToState:178(4160ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:25.030731Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518925343760361:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:24.897888Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518925758495690:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:24.959296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518919940681155:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:24.906066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518925758495698:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6487
2025-11-11T17:20:24.914096Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518925758495687:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:24.992727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518925758495681:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:24.992787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:24.992842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518925758495681:2109] [node 5] ICP22 created new session: [2:7571518925758495705:2109]
2025-11-11T17:20:24.992923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518925758495705:2109] [node 5] ICS09 handshake done sender: [2:7571518925758495687:2113] self: [2:7571518925758495686:0] peer: [5:7571518924235648530:0] socket: 454 qp: -1
2025-11-11T17:20:24.992963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518925758495705:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:24.993002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:24.993455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518925758495679:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:24.993490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:24.993533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518925758495679:2108] [node 6] ICP22 created new session: [2:7571518925758495707:2108]
2025-11-11T17:20:24.993578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518925758495707:2108] [node 6] ICS09 handshake done sender: [2:7571518925758495690:2115] self: [2:7571518925758495689:0] peer: [6:7571518925555810896:0] socket: 455 qp: -1
2025-11-11T17:20:24.993618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518925758495707:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:24.993642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:24.994177Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518925758495688:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:24.995738Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518925758495699:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.995923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518925758495704:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6487
2025-11-11T17:20:24.997018Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518925758495685:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:24.997327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518921463528319:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:24.997377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:24.997423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518921463528319:2048] [node 1] ICP22 created new session: [2:7571518925758495712:2048]
2025-11-11T17:20:24.997475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518925758495712:2048] [node 1] ICS09 handshake done sender: [2:7571518925758495688:2114] self: [2:7571518925758495694:0] peer: [1:7571518925209557489:0] socket: 452 qp: -1
2025-11-11T17:20:24.997508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518925758495712:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:24.997540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:25.026093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518921463528319:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28539841ns) {HandleHandshakeStatus:329(28536881ns) {DropIncomingHandshake:441(652ns) DropOutgoingHandshake:456(4012ns) SwitchToState:178(210ns) ProcessPendingSessionEvents:495(19807832ns) {ForwardSessionEventToSession:612(19746979ns) ForwardSessionEventToSession:612(10009ns) ForwardSessionEventToSession:612(3000ns) ForwardSessionEventToSession:612(5314ns) ForwardSessionEventToSession:612(2746ns) ForwardSessionEventToSession:612(3544ns) ForwardSessionEventToSession:612(5442ns) ForwardSessionEventToSession:612(3445ns) ForwardSessionEventToSession:612(8591ns) ForwardSessionEventToSession:612(4571ns)}}}
2025-11-11T17:20:25.040331Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:20:25.045761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518925758495680:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.045816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:25.045871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518925758495680:2107] [node 4] ICP22 created new session: [2:7571518930053463010:2107]
2025-11-11T17:20:25.045932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518930053463010:2107] [node 4] ICS09 handshake done sender: [2:7571518925758495685:2112] self: [2:7571518925758495684:0] peer: [4:7571518925343760369:0] socket: 453 qp: -1
2025-11-11T17:20:25.045970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518930053463010:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:25.046020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:25.098646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518925758495696:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.098690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518925758495696:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:25.109131Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518925758495696:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:24.959347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:24.959395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518919940681155:2050] [node 3] ICP22 created new session: [5:7571518924235648536:2050]
2025-11-11T17:20:24.959447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518924235648536:2050] [node 3] ICS09 handshake done sender: [5:7571518924235648520:2114] self: [5:7571518924235648529:0] peer: [3:7571518922847833027:0] socket: 459 qp: -1
2025-11-11T17:20:24.959498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518924235648536:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:24.959546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:24.960167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648534:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.970654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518924235648522:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(155485576ns) {Configure:104(155473004ns) {ProcessConfigured:135(80838ns) {StartInitialHandshake:157(77865ns) {DropHandshakes:469(7394ns) {DropIncomingHandshake:441(864ns) DropOutgoingHandshake:456(503ns)} GenerateSessionVirtualId:988(1491ns) PrepareNewSessionHandshake:476(2059ns) {SwitchToState:178(542ns)}}}}}
2025-11-11T17:20:24.970797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518924235648535:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:24.988562Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518924235648532:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.992053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518919940681153:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:25.001995Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:25.004670Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518924235648521:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:25.004986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518919940681154:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518924235648521:2115] held: no
2025-11-11T17:20:25.005093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518919940681154:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.005141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:25.005192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518919940681154:2049] [node 2] ICP22 created new session: [5:7571518924235648543:2049]
2025-11-11T17:20:25.005240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518924235648543:2049] [node 2] ICS09 handshake done sender: [5:7571518924235648521:2115] self: [5:7571518924235648530:0] peer: [2:7571518925758495686:0] socket: 460 qp: -1
2025-11-11T17:20:25.005278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518924235648543:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:25.005311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:25.078321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518924235648528:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:25.086045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518928530615842:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36213
2025-11-11T17:20:25.091814Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:25.094747Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518924235648540:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.094805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518924235648540:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:25.096093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518919940681153:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518924235648532:2119] held: no
2025-11-11T17:20:25.096212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518924235648511:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.096266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:25.096315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518924235648511:2108] [node 4] ICP22 created new session: [5:7571518928530615843:2108]
2025-11-11T17:20:25.096368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518928530615843:2108] [node 4] ICS09 handshake done sender: [5:7571518924235648528:2117] self: [5:7571518924235648533:0] peer: [4:7571518925343760360:0] socket: 476 qp: -1
2025-11-11T17:20:25.096420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518928530615843:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:25.096510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:25.097153Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518924235648532:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:25.102589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518919940681153:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.102642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:25.102692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518919940681153:2048] [node 1] ICP22 created new session: [5:7571518928530615846:2048]
2025-11-11T17:20:25.102742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518928530615846:2048] [node 1] ICS09 handshake done sender: [5:7571518924235648532:2119] self: [5:7571518924235648542:0] peer: [1:7571518925209557506:0] socket: 483 qp: -1
2025-11-11T17:20:25.102778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518928530615846:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:25.102810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:25.114558Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518924235648540:2122] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:25.043784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518925343760359:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.100401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518925209557494:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(27904913ns) {HandleHandshakeStatus:329(27902153ns) {DropIncomingHandshake:441(16660ns) DropOutgoingHandshake:456(614ns) SwitchToState:178(758ns) ProcessPendingSessionEvents:495(2642440ns) {ForwardSessionEventToSession:612(2588563ns) ForwardSessionEventToSession:612(5905ns) ForwardSessionEventToSession:612(7151ns) ForwardSessionEventToSession:612(6700ns) ForwardSessionEventToSession:612(2887ns) ForwardSessionEventToSession:612(5685ns) ForwardSessionEventToSession:612(4337ns) ForwardSessionEventToSession:612(3152ns) ForwardSessionEventToSession:612(5724ns) ForwardSessionEventToSession:612(2792ns)}}}
2025-11-11T17:20:24.883692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:24.910107Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:24.912524Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:24.917920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518921260843516:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518925555810887:2114] held: no
2025-11-11T17:20:24.918061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518921260843518:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518925555810882:2112] held: no
2025-11-11T17:20:24.918367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518925555810887:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:24.919187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518925555810882:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:24.942690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518921260843518:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:24.942782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:24.942845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518921260843518:2050] [node 3] ICP22 created new session: [6:7571518925555810900:2050]
2025-11-11T17:20:24.942897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518925555810900:2050] [node 3] ICS09 handshake done sender: [6:7571518925555810882:2112] self: [6:7571518925555810892:0] peer: [3:7571518922847833025:0] socket: 443 qp: -1
2025-11-11T17:20:24.942936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518925555810900:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:24.942999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:24.943498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518921260843516:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:24.943591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:24.943628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518921260843516:2048] [node 1] ICP22 created new session: [6:7571518925555810902:2048]
2025-11-11T17:20:24.943667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518925555810902:2048] [node 1] ICS09 handshake done sender: [6:7571518925555810887:2114] self: [6:7571518925555810891:0] peer: [1:7571518925209557502:0] socket: 456 qp: -1
2025-11-11T17:20:24.943694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518925555810902:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:24.943743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:24.944091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518925555810899:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:24.972421Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518925555810897:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:24.972982Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518925555810888:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:24.973371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518925555810879:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:24.973448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518921260843517:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:24.973494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:24.973537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518921260843517:2049] [node 2] ICP22 created new session: [6:7571518925555810907:2049]
2025-11-11T17:20:24.973619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518925555810907:2049] [node 2] ICS09 handshake done sender: [6:7571518925555810888:2115] self: [6:7571518925555810896:0] peer: [2:7571518925758495689:0] socket: 457 qp: -1
2025-11-11T17:20:24.973668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518925555810907:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:24.973711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:25.003698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518921260843517:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(30012707ns) {HandleHandshakeStatus:329(30011153ns) {DropIncomingHandshake:441(587ns) DropOutgoingHandshake:456(4182ns) SwitchToState:178(79ns) ProcessPendingSessionEvents:495(14947437ns) {ForwardSessionEventToSession:612(14884943ns) ForwardSessionEventToSession:612(13714ns) ForwardSessionEventToSession:612(3353ns) ForwardSessionEventToSession:612(2974ns) ForwardSessionEventToSession:612(5950ns) ForwardSessionEventToSession:612(3695ns) ForwardSessionEventToSession:612(7437ns) ForwardSessionEventToSession:612(3434ns) ForwardSessionEventToSession:612(3147ns) ForwardSessionEventToSession:612(5489ns)}}}
2025-11-11T17:20:25.048261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518929850778205:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:25.077568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518929850778206:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:25.082983Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518929850778205:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.086261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518925555810872:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:25.103644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518929850778208:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2666
2025-11-11T17:20:25.149731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518924235648522:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.149848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:25.149903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518924235648522:2109] [node 6] ICP22 created new session: [5:7571518928530615848:2109]
2025-11-11T17:20:25.149948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518928530615848:2109] [node 6] ICS09 handshake done sender: [5:7571518924235648540:2122] self: [5:7571518924235648539:0] peer: [6:7571518929850778207:0] socket: 462 qp: -1
2025-11-11T17:20:25.149986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518928530615848:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:25.150031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:25.043817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518925343760359:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:24.906499Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518922847833024:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:25.100593Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518925209557508:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:25.136862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518921463528320:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.109805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518920914590134:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.114003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:25.114058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518920914590134:2049] [node 3] ICP22 created new session: [1:7571518929504524828:2049]
2025-11-11T17:20:25.114112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518929504524828:2049] [node 3] ICS09 handshake done sender: [1:7571518925209557508:2115] self: [1:7571518925209557500:0] peer: [3:7571518927142800349:0] socket: 482 qp: -1
2025-11-11T17:20:25.114153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518929504524828:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:25.114187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:24.929924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518922847833018:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.044570Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:25.152324Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518925555810897:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:25.142105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:25.148989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518921463528320:2049] [node 3] ICP22 created new session: [2:7571518930053463013:2049]
2025-11-11T17:20:25.151730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518930053463013:2049] [node 3] ICS09 handshake done sender: [2:7571518925758495696:2116] self: [2:7571518925758495695:0] peer: [3:7571518927142800350:0] socket: 499 qp: -1
2025-11-11T17:20:25.158260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518930053463013:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:25.165804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:25.166317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518921463528320:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(38790470ns) {HandleHandshakeStatus:329(38787888ns) {DropIncomingHandshake:441(27741ns) DropOutgoingHandshake:456(496ns) SwitchToState:178(6393ns) ProcessPendingSessionEvents:495(124234ns) {ForwardSessionEventToSession:612(60583ns) ForwardSessionEventToSession:612(4351ns) ForwardSessionEventToSession:612(6272ns) ForwardSessionEventToSession:612(3030ns) ForwardSessionEventToSession:612(6193ns) ForwardSessionEventToSession:612(8329ns) ForwardSessionEventToSession:612(3770ns) ForwardSessionEventToSession:612(3778ns) ForwardSessionEventToSession:612(5124ns) ForwardSessionEventToSession:612(2698ns)}}}
2025-11-11T17:20:25.047890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518925343760357:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:24.945702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:25.157770Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518929850778205:2122] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:25.186040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518930053463015:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6487
2025-11-11T17:20:25.186154Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:20:24.949652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518922847833049:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-11-11T17:20:25.074950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518921048792993:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518925343760357:2112] held: no
2025-11-11T17:20:25.165277Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:24.954471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518922847833018:2107] [node 4] ICP22 created new session: [3:7571518922847833050:2107]
2025-11-11T17:20:25.075071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518921048792993:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.168495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518925555810872:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518929850778205:2122] held: no
2025-11-11T17:20:25.075117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:25.075171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518921048792993:2048] [node 1] ICP22 created new session: [4:7571518929638727676:2048]
2025-11-11T17:20:25.075220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518929638727676:2048] [node 1] ICS09 handshake done sender: [4:7571518925343760357:2112] self: [4:7571518925343760372:0] peer: [1:7571518925209557496:0] socket: 451 qp: -1
2025-11-11T17:20:25.075255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518929638727676:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:25.075289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:25.075593Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518925343760356:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:25.076129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518925343760351:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.076171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:25.076220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518925343760351:2108] [node 5] ICP22 created new session: [4:7571518929638727678:2108]
2025-11-11T17:20:25.076269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518929638727678:2108] [node 5] ICS09 handshake done sender: [4:7571518925343760361:2114] self: [4:7571518925343760360:0] peer: [5:7571518924235648533:0] socket: 474 qp: -1
2025-11-11T17:20:25.076306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518929638727678:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:25.076375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:25.077107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518921048792995:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.077176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:25.082746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518921048792995:2050] [node 3] ICP22 created new session: [4:7571518929638727680:2050]
2025-11-11T17:20:25.088933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518929638727680:2050] [node 3] ICS09 handshake done sender: [4:7571518925343760356:2111] self: [4:7571518925343760371:0] peer: [3:7571518922847833023:0] socket: 450 qp: -1
2025-11-11T17:20:25.089005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518929638727680:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:25.097565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:25.097955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518929638727672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
2025-11-11T17:20:25.098100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518929638727674:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
2025-11-11T17:20:25.111096Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:25.135722Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518925343760359:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:25.165776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518929638727673:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6448
2025-11-11T17:20:25.168047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518925343760351:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:25.174116Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518925343760364:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:25.174275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518921048792995:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(76332555ns) {HandleHandshakeStatus:329(76329685ns) {DropIncomingHandshake:441(442ns) DropOutgoingHandshake:456(6514ns) SwitchToState:178(132ns) ProcessPendingSessionEvents:495(33303936ns) {ForwardSessionEventToSession:612(7066144ns) ForwardSessionEventToSession:612(6757ns) ForwardSessionEventToSession:612(10573222ns) ForwardSessionEventToSession:612(5102ns) ForwardSessionEventToSession:612(9359975ns) ForwardSessionEventToSession:612(10784ns) ForwardSessionEventToSession:612(2054306ns) ForwardSessionEventToSession:612(4178138ns) ForwardSessionEventToSession:612(14459ns) ForwardSessionEventToSession:612(4025ns)}}}
2025-11-11T17:20:25.179041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518925343760353:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:25.183787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518929638727673:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.184360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518921048792994:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.184455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:25.184505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518921048792994:2049] [node 2] ICP22 created new session: [4:7571518929638727684:2049]
2025-11-11T17:20:25.184579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518929638727684:2049] [node 2] ICS09 handshake done sender: [4:7571518925343760364:2116] self: [4:7571518925343760369:0] peer: [2:7571518925758495684:0] socket: 471 qp: -1
2025-11-11T17:20:25.184618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518929638727684:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:25.184648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:25.186852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:25.192739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518925343760353:2109] [node 6] ICP22 created new session: [4:7571518929638727686:2109]
2025-11-11T17:20:25.203350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518929638727686:2109] [node 6] ICS09 handshake done sender: [4:7571518925343760359:2113] self: [4:7571518925343760358:0] peer: [6:7571518925555810906:0] socket: 473 qp: -1
2025-11-11T17:20:25.205144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518929638727686:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:25.211316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:24.974285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518922847833050:2107] [node 4] ICS09 handshake done sender: [3:7571518922847833024:2111] self: [3:7571518922847833023:0] peer: [4:7571518925343760371:0] socket: 439 qp: -1
2025-11-11T17:20:25.169186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518925555810872:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.169244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:25.169295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518925555810872:2106] [node 5] ICP22 created new session: [6:7571518929850778209:2106]
2025-11-11T17:20:25.169337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518929850778209:2106] [node 5] ICS09 handshake done sender: [6:7571518929850778205:2122] self: [6:7571518929850778207:0] peer: [5:7571518924235648539:0] socket: 495 qp: -1
2025-11-11T17:20:25.169370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518929850778209:2106] [node 5] ICS10 traffic start
2025-11-11T17:20:25.169403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:25.170223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518925555810879:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.196227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:25.201961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518925555810879:2110] [node 4] ICP22 created new session: [6:7571518929850778211:2110]
2025-11-11T17:20:25.203624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518929850778211:2110] [node 4] ICS09 handshake done sender: [6:7571518925555810897:2119] self: [6:7571518925555810906:0] peer: [4:7571518925343760358:0] socket: 475 qp: -1
2025-11-11T17:20:25.203673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518929850778211:2110] [node 4] ICS10 traffic start
2025-11-11T17:20:25.205922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:25.217247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518925555810879:2110] [node 4] ICP35 event processing took too much time PendingConnection:278(46778552ns) {HandleHandshakeStatus:329(46776477ns) {DropIncomingHandshake:441(352ns) DropOutgoingHandshake:456(4470ns) SwitchToState:178(7996ns) ProcessPendingSessionEvents:495(3463327ns) {ForwardSessionEventToSession:612(3389071ns) ForwardSessionEventToSession:612(4828ns) ForwardSessionEventToSession:612(6009ns) ForwardSessionEventToSession:612(12456ns) ForwardSessionEventToSession:612(2691ns) ForwardSessionEventToSession:612(5294ns) ForwardSessionEventToSession:612(3435ns) ForwardSessionEventToSession:612(14896ns) ForwardSessionEventToSession:612(11211ns) ForwardSessionEventToSession:612(2630ns)}}}
2025-11-11T17:20:25.214291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518925343760353:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(35012635ns) {HandleHandshakeStatus:329(35008397ns) {DropIncomingHandshake:441(441ns) DropOutgoingHandshake:456(692ns) SwitchToState:178(464ns) ProcessPendingSessionEvents:495(110452ns) {ForwardSessionEventToSession:612(58602ns) ForwardSessionEventToSession:612(6279ns) ForwardSessionEventToSession:612(5431ns) ForwardSessionEventToSession:612(3016ns) ForwardSessionEventToSession:612(5631ns) ForwardSessionEventToSession:612(2479ns) ForwardSessionEventToSession:612(2962ns) ForwardSessionEventToSession:612(5883ns) ForwardSessionEventToSession:612(3104ns) ForwardSessionEventToSession:612(5491ns)}}}
2025-11-11T17:20:25.214410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518925343760353:2109] [node 6] ICP18 handshake (actor [4:7571518929638727673:2121]) is too old
2025-11-11T17:20:24.996823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518922847833051:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-11-11T17:20:24.997856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518922847833050:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:25.013914Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518922847833049:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.019350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:25.029701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518918552865660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:25.066031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518922847833018:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(135890624ns) {HandleHandshakeStatus:329(135885963ns) {DropIncomingHandshake:441(662ns) DropOutgoingHandshake:456(1064ns) SwitchToState:178(597ns) ProcessPendingSessionEvents:495(17192809ns) {ForwardSessionEventToSession:612(17117955ns) ForwardSessionEventToSession:612(12841ns) ForwardSessionEventToSession:612(3415ns) ForwardSessionEventToSession:612(4438ns) ForwardSessionEventToSession:612(6564ns) ForwardSessionEventToSession:612(2884ns) ForwardSessionEventToSession:612(5719ns) ForwardSessionEventToSession:612(12385ns) ForwardSessionEventToSession:612(3722ns) ForwardSessionEventToSession:612(5744ns)}}}
2025-11-11T17:20:25.073743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518922847833051:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:25.075816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518918552865661:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:25.108875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518927142800352:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-11-11T17:20:25.120275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518927142800351:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-11-11T17:20:25.148688Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518922847833051:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:25.151789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518922847833049:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:25.168753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518918552865661:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.168801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:25.170446Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:25.173674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518918552865661:2049] [node 2] ICP22 created new session: [3:7571518927142800353:2049]
2025-11-11T17:20:25.173781Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:25.178805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518927142800353:2049] [node 2] ICS09 handshake done sender: [3:7571518922847833051:2119] self: [3:7571518927142800350:0] peer: [2:7571518925758495695:0] socket: 500 qp: -1
2025-11-11T17:20:25.183337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518927142800353:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:25.183398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:25.202586Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518918552865661:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(34323853ns) {HandleHandshakeStatus:329(34321164ns) {DropIncomingHandshake:441(853ns) DropOutgoingHandshake:456(3049ns) SwitchToState:178(412ns) ProcessPendingSessionEvents:495(138653ns) {ForwardSessionEventToSession:612(85308ns) ForwardSessionEventToSession:612(3946ns) ForwardSessionEventToSession:612(2590ns) ForwardSessionEventToSession:612(5608ns) ForwardSessionEventToSession:612(3351ns) ForwardSessionEventToSession:612(2755ns) ForwardSessionEventToSession:612(6964ns) ForwardSessionEventToSession:612(3826ns) ForwardSessionEventToSession:612(8671ns) ForwardSessionEventToSession:612(4033ns)}}}
2025-11-11T17:20:25.202659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518918552865660:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:25.202717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:25.202759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518918552865660:2048] [node 1] ICP22 created new session: [3:7571518927142800355:2048]
2025-11-11T17:20:25.202800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518927142800355:2048] [node 1] ICS09 handshake done sender: [3:7571518922847833049:2118] self: [3:7571518927142800349:0] peer: [1:7571518925209557500:0] socket: 488 qp: -1
2025-11-11T17:20:25.202839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518927142800355:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:25.202876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:25.204386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518918552865661:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:25.216403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518918552865660:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:25.277149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:25.277397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518925758495712:2048] [node 1] ICS07 socket disconnect 452 reason# EndOfStream
2025-11-11T17:20:25.277426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518925758495712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.277514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:25.277554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518925758495712:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:25.287084Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:20:25.293719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:25.294928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518927142800355:2048] [node 1] ICS07 socket disconnect 488 reason# EndOfStream
2025-11-11T17:20:25.294963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518927142800355:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.295074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:25.295123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518927142800355:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:25.294189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518921463528319:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:20:25.294237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518925758495712:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:25.299645Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address
2025-11-11T17:20:25.299937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518921463528319:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:25.299981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518925758495712:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:25.300010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518921463528319:2048] [node 1] ICP30 unregister session Session# [2:7571518925758495712:2048] VirtualId# [2:7571518925758495694:0]
2025-11-11T17:20:25.300049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:25.300103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518921463528319:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.300118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.298822Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address
2025-11-11T17:20:25.299147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518918552865660:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:25.299201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518927142800355:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:25.299238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518918552865660:2048] [node 1] ICP30 unregister session Session# [3:7571518927142800355:2048] VirtualId# [3:7571518927142800349:0]
2025-11-11T17:20:25.299291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:25.299322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518918552865660:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.299338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.305398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:25.305660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518928530615846:2048] [node 1] ICS01 socket: 483 reason# EndOfStream
2025-11-11T17:20:25.305709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518919940681153:2048] [node 1] ICP30 unregister session Session# [5:7571518928530615846:2048] VirtualId# [5:7571518924235648542:0]
2025-11-11T17:20:25.305742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518928530615846:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.305833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:25.305879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:25.305695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:25.305902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518929638727676:2048] [node 1] ICS01 socket: 451 reason# EndOfStream
2025-11-11T17:20:25.305935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518921048792993:2048] [node 1] ICP30 unregister session Session# [4:7571518929638727676:2048] VirtualId# [4:7571518925343760372:0]
2025-11-11T17:20:25.305969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518929638727676:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.306039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:25.306080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:25.306253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:25.306450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518925555810902:2048] [node 1] ICS01 socket: 456 reason# EndOfStream
2025-11-11T17:20:25.306496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518921260843516:2048] [node 1] ICP30 unregister session Session# [6:7571518925555810902:2048] VirtualId# [6:7571518925555810891:0]
2025-11-11T17:20:25.306521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518925555810902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.306575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:25.306605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:25.311824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518921463528319:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:25.313711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518918552865660:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:25.385931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:25.386154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571518927142800353:2049] [node 2] ICS07 socket disconnect 500 reason# EndOfStream
2025-11-11T17:20:25.386186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518927142800353:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.386305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:25.386409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571518927142800353:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:25.397825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:25.398037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518925555810907:2049] [node 2] ICS07 socket disconnect 457 reason# EndOfStream
2025-11-11T17:20:25.398068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518925555810907:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.398165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:25.398213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518925555810907:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:25.398744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:25.398922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518924235648543:2049] [node 2] ICS01 socket: 460 reason# EndOfStream
2025-11-11T17:20:25.398961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518919940681154:2049] [node 2] ICP30 unregister session Session# [5:7571518924235648543:2049] VirtualId# [5:7571518924235648530:0]
2025-11-11T17:20:25.398995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518924235648543:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.399059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:25.399106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:25.400025Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address
2025-11-11T17:20:25.400327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518918552865661:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:25.400385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518927142800353:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:25.400420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518918552865661:2049] [node 2] ICP30 unregister session Session# [3:7571518927142800353:2049] VirtualId# [3:7571518927142800350:0]
2025-11-11T17:20:25.400469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:25.400520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571518918552865661:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.400539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.401095Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address
2025-11-11T17:20:25.401371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518921260843517:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:25.401415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518925555810907:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:25.401449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518921260843517:2049] [node 2] ICP30 unregister session Session# [6:7571518925555810907:2049] VirtualId# [6:7571518925555810896:0]
2025-11-11T17:20:25.401491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:25.401533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518921260843517:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.401546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.402230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:25.402504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518929638727684:2049] [node 2] ICS01 socket: 471 reason# EndOfStream
2025-11-11T17:20:25.402544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518921048792994:2049] [node 2] ICP30 unregister session Session# [4:7571518929638727684:2049] VirtualId# [4:7571518925343760369:0]
2025-11-11T17:20:25.402575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518929638727684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.402673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:25.402719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:25.413751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571518918552865661:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:25.414350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518921260843517:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:25.537805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:25.538019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518925555810900:2050] [node 3] ICS01 socket: 443 reason# EndOfStream
2025-11-11T17:20:25.538072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518921260843518:2050] [node 3] ICP30 unregister session Session# [6:7571518925555810900:2050] VirtualId# [6:7571518925555810892:0]
2025-11-11T17:20:25.538104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518925555810900:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.538179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:25.538223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:25.538100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:25.538322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518929638727680:2050] [node 3] ICS07 socket disconnect 450 reason# EndOfStream
2025-11-11T17:20:25.538355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518929638727680:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.538454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:25.538514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518929638727680:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:25.541712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:25.539202Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Couldn't connect to any resolved address
2025-11-11T17:20:25.541973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518924235648536:2050] [node 3] ICS01 socket: 459 reason# EndOfStream
2025-11-11T17:20:25.539471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518921048792995:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:25.539517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518929638727680:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:25.542014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518919940681155:2050] [node 3] ICP30 unregister session Session# [5:7571518924235648536:2050] VirtualId# [5:7571518924235648529:0]
2025-11-11T17:20:25.539563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518921048792995:2050] [node 3] ICP30 unregister session Session# [4:7571518929638727680:2050] VirtualId# [4:7571518925343760371:0]
2025-11-11T17:20:25.542051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518924235648536:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.539609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:25.542152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:25.539645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518921048792995:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.542202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:25.539678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17851) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.552989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518921048792995:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:20:25.677770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:25.678002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518929850778211:2110] [node 4] ICS07 socket disconnect 475 reason# EndOfStream
2025-11-11T17:20:25.678034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518929850778211:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.678132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:25.678184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518929850778211:2110] [node 4] ICS15 start handshake
2025-11-11T17:20:25.678890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:25.678912Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12313) Couldn't connect to any resolved address
2025-11-11T17:20:25.679131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518928530615843:2108] [node 4] ICS01 socket: 476 reason# EndOfStream
2025-11-11T17:20:25.679154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518925555810879:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:25.679175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518924235648511:2108] [node 4] ICP30 unregister session Session# [5:7571518928530615843:2108] VirtualId# [5:7571518924235648533:0]
2025-11-11T17:20:25.679202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518929850778211:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:25.679210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518928530615843:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.679242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518925555810879:2110] [node 4] ICP30 unregister session Session# [6:7571518929850778211:2110] VirtualId# [6:7571518925555810906:0]
2025-11-11T17:20:25.679284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:25.679285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:25.679329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:25.679321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518925555810879:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.679336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:25.693710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518925555810879:2110] [node 4] ICP33 wake up from error state
2025-11-11T17:20:25.746640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:25.746975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518929850778209:2106] [node 5] ICS01 socket: 495 reason# EndOfStream
2025-11-11T17:20:25.747051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518925555810872:2106] [node 5] ICP30 unregister session Session# [6:7571518929850778209:2106] VirtualId# [6:7571518929850778207:0]
2025-11-11T17:20:25.747111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518929850778209:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:25.747268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:25.747348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:25.945469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518927287282008:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:25.945520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518927287282009:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:25.977434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518929455607827:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:25.977479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518929455607828:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:26.017765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518928567937795:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:26.017831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518928567937796:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:26.059152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518930846957755:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:26.059203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518930846957756:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:26.059713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518930846957754:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:26.087927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518931304477995:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:26.087974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518931304477996:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:26.088004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518931304477997:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:26.104498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518931224739315:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:26.104538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518931224739316:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:26.104599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518931224739317:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:26.476304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518931224739366:2096] [node 5] ICP01 ready to work
2025-11-11T17:20:26.476999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518931224739367:2093] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.545159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518931304478056:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:26.542928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518931224739366:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(66325966ns) {Configure:104(66320980ns) {ProcessConfigured:135(81869ns) {StartInitialHandshake:157(79331ns) {DropHandshakes:469(2680ns) {DropIncomingHandshake:441(575ns) DropOutgoingHandshake:456(651ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:476(1143ns) {SwitchToState:178(167ns)}}}}}
2025-11-11T17:20:26.562195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518931582249367:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:26.564079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518933750575184:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:26.590544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518931582249368:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:26.669829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518932862905163:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:26.671495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518933750575187:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:26.673686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518930846957817:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:26.653203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518931582249369:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:26.676755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518933750575188:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:26.676772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518932862905164:2107] [node 6] ICP01 ready to work
2025-11-11T17:20:26.674373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518931304478060:2110] [node 6] ICP01 ready to work
2025-11-11T17:20:26.681671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518931304478059:2109] [node 4] ICP01 ready to work
2025-11-11T17:20:26.739079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518931304477997:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(68852755ns) {Configure:104(68847777ns) {ProcessConfigured:135(75560ns) {StartInitialHandshake:157(72245ns) {DropHandshakes:469(2823ns) {DropIncomingHandshake:441(596ns) DropOutgoingHandshake:456(763ns)} GenerateSessionVirtualId:988(796ns) PrepareNewSessionHandshake:476(1852ns) {SwitchToState:178(704ns)}}}}}
2025-11-11T17:20:26.739289Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518931304478056:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.720665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518932862905167:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:26.722087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518930846957818:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:26.722369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518931224739367:2093] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(245065070ns) {Configure:104(245061261ns) {ProcessConfigured:135(17798375ns) {StartInitialHandshake:157(17795054ns) {DropHandshakes:469(2693ns) {DropIncomingHandshake:441(520ns) DropOutgoingHandshake:456(513ns)} GenerateSessionVirtualId:988(792ns) PrepareNewSessionHandshake:476(5089ns) {SwitchToState:178(2654ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.802023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518931304477996:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(70205234ns) {Configure:104(70201091ns) {ProcessConfigured:135(35177109ns) {StartInitialHandshake:157(35174537ns) {DropHandshakes:469(2256ns) {DropIncomingHandshake:441(488ns) DropOutgoingHandshake:456(349ns)} GenerateSessionVirtualId:988(1033ns) PrepareNewSessionHandshake:476(2684ns) {SwitchToState:178(1024ns)}}}}}
2025-11-11T17:20:26.764127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518930846957754:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(201913454ns) {Configure:104(201906490ns) {ProcessConfigured:135(71608ns) {StartInitialHandshake:157(68813ns) {DropHandshakes:469(2659ns) {DropIncomingHandshake:441(485ns) DropOutgoingHandshake:456(786ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:476(1787ns) {SwitchToState:178(741ns)}}}}}
2025-11-11T17:20:26.790541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518929455607827:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(300453573ns) {Configure:104(292629988ns) {ProcessConfigured:135(65878ns) {StartInitialHandshake:157(60222ns) {DropHandshakes:469(2664ns) {DropIncomingHandshake:441(722ns) DropOutgoingHandshake:456(483ns)} GenerateSessionVirtualId:988(1011ns) PrepareNewSessionHandshake:476(2097ns) {SwitchToState:178(611ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.825484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518933750575189:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46377
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.825735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518933750575196:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46377
2025-11-11T17:20:26.798162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518931582249367:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(82942740ns) {Configure:104(82932660ns) {ProcessConfigured:135(662648ns) {StartInitialHandshake:157(658825ns) {DropHandshakes:469(3283ns) {DropIncomingHandshake:441(943ns) DropOutgoingHandshake:456(566ns)} GenerateSessionVirtualId:988(957ns) PrepareNewSessionHandshake:476(3630ns) {SwitchToState:178(2283ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.837646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518931224739389:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.779458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518930846957756:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(101595294ns) {Configure:104(101588387ns) {ProcessConfigured:135(76708ns) {StartInitialHandshake:157(74253ns) {DropHandshakes:469(2596ns) {DropIncomingHandshake:441(818ns) DropOutgoingHandshake:456(436ns)} GenerateSessionVirtualId:988(1211ns) PrepareNewSessionHandshake:476(2410ns) {SwitchToState:178(1063ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.800259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518931582249374:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20512
2025-11-11T17:20:26.842105Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518933750575189:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.872068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518932862905176:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47487
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.892471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518930846957828:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
2025-11-11T17:20:26.892669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518930846957829:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.892829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518930846957830:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
2025-11-11T17:20:26.892951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518930846957831:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
2025-11-11T17:20:26.868083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518931224739391:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.898429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518931304478070:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:26.802887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518927287282008:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(218071611ns) {Configure:104(212035240ns) {ProcessConfigured:135(65828ns) {StartInitialHandshake:157(63585ns) {DropHandshakes:469(1866ns) {DropIncomingHandshake:441(506ns) DropOutgoingHandshake:456(344ns)} GenerateSessionVirtualId:988(743ns) PrepareNewSessionHandshake:476(1346ns) {SwitchToState:178(483ns)}}}}}
2025-11-11T17:20:26.898970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518931304478072:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:26.806342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518927287282009:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(297972095ns) {Configure:104(297967019ns) {ProcessConfigured:135(54444ns) {StartInitialHandshake:157(51794ns) {DropHandshakes:469(2071ns) {DropIncomingHandshake:441(333ns) DropOutgoingHandshake:456(362ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:476(1350ns) {SwitchToState:178(432ns)}}}}}
2025-11-11T17:20:26.862008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518931582249383:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20512
2025-11-11T17:20:26.865891Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249374:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.889660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518931582249382:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20512
2025-11-11T17:20:26.893655Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249383:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.870234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518933750575197:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46377
2025-11-11T17:20:26.902046Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518930846957829:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.902179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518930846957828:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.902573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518930846957756:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:26.909883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518931304477995:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(308954400ns) {Configure:104(308950223ns) {ProcessConfigured:135(42356730ns) {StartInitialHandshake:157(42354080ns) {DropHandshakes:469(2634ns) {DropIncomingHandshake:441(659ns) DropOutgoingHandshake:456(467ns)} GenerateSessionVirtualId:988(1014ns) PrepareNewSessionHandshake:476(2336ns) {SwitchToState:178(773ns)}}}}}
2025-11-11T17:20:26.872307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518932862905175:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47487
2025-11-11T17:20:26.899694Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518931224739391:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.883900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518928567937796:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(209845743ns) {Configure:104(209835741ns) {ProcessConfigured:135(139157ns) {StartInitialHandshake:157(135884ns) {DropHandshakes:469(6639ns) {DropIncomingHandshake:441(771ns) DropOutgoingHandshake:456(379ns)} GenerateSessionVirtualId:988(830ns) PrepareNewSessionHandshake:476(2836ns) {SwitchToState:178(999ns)}}}}}
2025-11-11T17:20:26.887814Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518933750575196:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.926983Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518930846957831:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.929825Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249382:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.900085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518931224739388:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:26.898811Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518932862905175:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.930327Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518930846957830:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.935346Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518931304478072:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.893500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518929455607828:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(230583446ns) {Configure:104(230573820ns) {ProcessConfigured:135(22782254ns) {StartInitialHandshake:157(22778264ns) {DropHandshakes:469(3268ns) {DropIncomingHandshake:441(646ns) DropOutgoingHandshake:456(659ns)} GenerateSessionVirtualId:988(740ns) PrepareNewSessionHandshake:476(4017ns) {SwitchToState:178(1415ns)}}}}}
2025-11-11T17:20:26.913190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518931224739317:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(250433510ns) {Configure:104(250425275ns) {ProcessConfigured:135(12849466ns) {StartInitialHandshake:157(12845418ns) {DropHandshakes:469(1556ns) {DropIncomingHandshake:441(316ns) DropOutgoingHandshake:456(601ns)} GenerateSessionVirtualId:988(345ns) PrepareNewSessionHandshake:476(12837ns) {SwitchToState:178(1814ns)}}}}}
2025-11-11T17:20:26.921794Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518932862905176:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.942290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518931304478060:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(232448529ns) {Configure:104(232437642ns) {ProcessConfigured:135(11592534ns) {StartInitialHandshake:157(11590465ns) {DropHandshakes:469(2183ns) {DropIncomingHandshake:441(254ns) DropOutgoingHandshake:456(406ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:476(4015ns) {SwitchToState:178(2129ns)}}}}}
2025-11-11T17:20:26.965341Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249371:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.937870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518930846957817:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(199798719ns) {Configure:104(199793277ns) {ProcessConfigured:135(22603918ns) {StartInitialHandshake:157(22598552ns) {DropHandshakes:469(2303ns) {DropIncomingHandshake:441(227ns) DropOutgoingHandshake:456(635ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:476(5485ns) {SwitchToState:178(3269ns)}}}}}
No RDMA context found for GID: 2025-11-11T17:20:26.925089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518932862905173:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-11T17:20:26.913466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518931224739316:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:26.916609Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518933750575197:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.965399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518931582249371:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:26.946057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518931304477996:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:26.925143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518932862905173:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:26.946362Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518932862905173:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:26.978291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518932862905167:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(246484995ns) {Configure:104(235652917ns) {ProcessConfigured:135(4947461ns) {StartInitialHandshake:157(4944603ns) {DropHandshakes:469(2620ns) {DropIncomingHandshake:441(763ns) DropOutgoingHandshake:456(374ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:476(3415ns) {SwitchToState:178(1489ns)}}}}}
2025-11-11T17:20:27.002630Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518932862905169:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.002678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518932862905169:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.004589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518932862905163:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.004693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:27.004747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518932862905163:2108] [node 4] ICP22 created new session: [3:7571518937157872480:2108]
2025-11-11T17:20:27.004800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518937157872480:2108] [node 4] ICS09 handshake done sender: [3:7571518932862905173:2121] self: [3:7571518932862905172:0] peer: [4:7571518930846957834:0] socket: 453 qp: -1
2025-11-11T17:20:27.004835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518937157872480:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:26.942206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518930846957755:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(270043590ns) {Configure:104(270038753ns) {ProcessConfigured:135(42288740ns) {StartInitialHandshake:157(42286071ns) {DropHandshakes:469(1249ns) {DropIncomingHandshake:441(249ns) DropOutgoingHandshake:456(276ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:476(2953ns) {SwitchToState:178(1869ns)}}}}}
2025-11-11T17:20:26.914072Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518931224739389:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.004907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:26.947717Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518931304478070:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.946146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518929455607827:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:26.968036Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249378:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.947666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518930846957754:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.973132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518931304477995:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:26.939006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518931224739367:2093] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:26.962335Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518930846957824:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.949549Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518933750575195:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.045368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518935599445376:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:26.968082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518931582249378:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.018047Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518932862905169:2120] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.063969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518935599445377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:27.064777Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518931304478072:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:26.942211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518931224739367:2093] [node 4] ICP35 event processing took too much time PendingConnection:278(17237131ns) {IncomingHandshake:261(17234940ns) {DropIncomingHandshake:441(459ns) PrepareNewSessionHandshake:476(1511ns) {SwitchToState:178(586ns)} GenerateSessionVirtualId:988(480ns) IssueIncomingHandshakeReply:193(68766ns) {SwitchToState:178(571ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.962371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518930846957824:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:26.970862Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518931582249371:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:27.023225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518937157872484:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47487
2025-11-11T17:20:26.949609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518933750575195:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.971226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518931582249367:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.065006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518931304477996:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.974828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518931224739398:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:26.990005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518930846957824:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:27.026895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518932862905164:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:26.962112Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518933750575195:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:27.065093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:26.971291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:26.975353Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518931224739388:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.967983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518933750575184:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(219030791ns) {Configure:104(219019033ns) {ProcessConfigured:135(33246630ns) {StartInitialHandshake:157(33244338ns) {DropHandshakes:469(2663ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:476(1692ns) {SwitchToState:178(146ns)}}}}}
2025-11-11T17:20:26.971344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518931582249367:2107] [node 4] ICP22 created new session: [1:7571518931582249390:2107]
2025-11-11T17:20:26.971386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518931582249390:2107] [node 4] ICS09 handshake done sender: [1:7571518931582249371:2110] self: [1:7571518931582249370:0] peer: [4:7571518930846957839:0] socket: 460 qp: -1
2025-11-11T17:20:26.971421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518931582249390:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:26.971458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:26.978635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518931582249369:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(275653404ns) {Configure:104(275646041ns) {ProcessConfigured:135(50943326ns) {StartInitialHandshake:157(50941048ns) {DropHandshakes:469(14468194ns) {DropIncomingHandshake:441(2121ns) DropOutgoingHandshake:456(462ns)} GenerateSessionVirtualId:988(1696ns) PrepareNewSessionHandshake:476(1563ns) {SwitchToState:178(141ns)}}}}}
2025-11-11T17:20:26.990416Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249373:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:26.990443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518931582249373:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:26.991253Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518931582249373:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:27.018158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518931582249378:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:27.021730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518931582249368:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.021816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:27.021880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518931582249368:2108] [node 5] ICP22 created new session: [1:7571518935877216691:2108]
2025-11-11T17:20:27.021930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518935877216691:2108] [node 5] ICS09 handshake done sender: [1:7571518931582249373:2111] self: [1:7571518931582249372:0] peer: [5:7571518931304478078:0] socket: 461 qp: -1
2025-11-11T17:20:27.021968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518935877216691:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:27.022012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:27.031942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518927287282008:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.031999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:27.049634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518927287282008:2048] [node 2] ICP22 created new session: [1:7571518935877216694:2048]
2025-11-11T17:20:27.057800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518935877216694:2048] [node 2] ICS09 handshake done sender: [1:7571518931582249378:2113] self: [1:7571518931582249377:0] peer: [2:7571518933750575207:0] socket: 462 qp: -1
2025-11-11T17:20:27.066516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518935877216694:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:27.071594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:27.081709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249381:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.081748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518931582249381:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.107083Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518931582249388:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.107107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518931582249388:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.108738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518931582249388:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:27.130041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518935877216696:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20512
2025-11-11T17:20:27.131123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518931582249369:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.002593Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:27.026956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:27.165752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518935877216696:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.165818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:27.065147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518931304477996:2049] [node 2] ICP22 created new session: [5:7571518935599445378:2049]
2025-11-11T17:20:26.977810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518931224739315:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(287557861ns) {Configure:104(287550774ns) {ProcessConfigured:135(46558016ns) {StartInitialHandshake:157(46555312ns) {DropHandshakes:469(2695ns) {DropIncomingHandshake:441(935ns) DropOutgoingHandshake:456(412ns)} GenerateSessionVirtualId:988(818ns) PrepareNewSessionHandshake:476(5105ns) {SwitchToState:178(2323ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.003423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518930846957840:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.009796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518933750575187:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.173408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518931582249369:2109] [node 6] ICP22 created new session: [1:7571518935877216698:2109]
2025-11-11T17:20:27.173752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518931582249381:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:27.185539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518935877216698:2109] [node 6] ICS09 handshake done sender: [1:7571518931582249388:2117] self: [1:7571518931582249386:0] peer: [6:7571518935519706706:0] socket: 468 qp: -1
2025-11-11T17:20:27.065193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518935599445378:2049] [node 2] ICS09 handshake done sender: [5:7571518931304478072:2115] self: [5:7571518931304478076:0] peer: [2:7571518933750575192:0] socket: 479 qp: -1
2025-11-11T17:20:27.030432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518938045542505:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46377
2025-11-11T17:20:27.031596Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518933750575193:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.036203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:27.036267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518933750575187:2109] [node 6] ICP22 created new session: [2:7571518938045542507:2109]
2025-11-11T17:20:27.041698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518933750575193:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.049751Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518933750575197:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:27.051814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518929455607827:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.051850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:27.051888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518929455607827:2048] [node 1] ICP22 created new session: [2:7571518938045542509:2048]
2025-11-11T17:20:27.051926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518938045542509:2048] [node 1] ICS09 handshake done sender: [2:7571518933750575197:2115] self: [2:7571518933750575207:0] peer: [1:7571518931582249377:0] socket: 464 qp: -1
2025-11-11T17:20:27.051952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518938045542509:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:27.051977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:27.052410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518938045542508:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46377
2025-11-11T17:20:27.064222Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518933750575193:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:27.074655Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:27.074756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518938045542507:2109] [node 6] ICS09 handshake done sender: [2:7571518933750575195:2113] self: [2:7571518933750575194:0] peer: [6:7571518931224739394:0] socket: 471 qp: -1
2025-11-11T17:20:27.087805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518933750575188:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.087874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:27.087931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518933750575188:2108] [node 5] ICP22 created new session: [2:7571518938045542512:2108]
2025-11-11T17:20:27.087985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518938045542512:2108] [node 5] ICS09 handshake done sender: [2:7571518933750575193:2112] self: [2:7571518933750575192:0] peer: [5:7571518931304478076:0] socket: 472 qp: -1
2025-11-11T17:20:27.088024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518938045542512:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:27.088059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:27.088598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518929455607827:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:27.081692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518938045542507:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:27.102380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:27.133861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518938045542514:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46377
2025-11-11T17:20:27.134337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518933750575187:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(127156987ns) {HandleHandshakeStatus:329(127154454ns) {DropIncomingHandshake:441(20073ns) DropOutgoingHandshake:456(431ns) SwitchToState:178(974ns) ProcessPendingSessionEvents:495(112169ns) {ForwardSessionEventToSession:612(56036ns) ForwardSessionEventToSession:612(6477ns) ForwardSessionEventToSession:612(3112ns) ForwardSessionEventToSession:612(5860ns) ForwardSessionEventToSession:612(2651ns) ForwardSessionEventToSession:612(5570ns) ForwardSessionEventToSession:612(11433ns) ForwardSessionEventToSession:612(3118ns) ForwardSessionEventToSession:612(2569ns) ForwardSessionEventToSession:612(4837ns)}}}
2025-11-11T17:20:27.134496Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518938045542508:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.135759Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518933750575204:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.135798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518933750575204:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.170186Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518933750575204:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:27.027005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518932862905164:2107] [node 6] ICP22 created new session: [3:7571518937157872485:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.018679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518930846957754:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518930846957830:2115] held: no
2025-11-11T17:20:26.981830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518931224739317:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.197657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518935877216698:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:27.202931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518927287282009:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.203038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:27.203088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518927287282009:2049] [node 3] ICP22 created new session: [1:7571518935877216700:2049]
2025-11-11T17:20:27.203135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518935877216700:2049] [node 3] ICS09 handshake done sender: [1:7571518931582249381:2114] self: [1:7571518931582249380:0] peer: [3:7571518937157872489:0] socket: 490 qp: -1
2025-11-11T17:20:27.203173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518935877216700:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:27.203214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:27.205693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:27.209619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518927287282008:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(133655401ns) {HandleHandshakeStatus:329(133652826ns) {DropIncomingHandshake:441(14709ns) DropOutgoingHandshake:456(638ns) SwitchToState:178(917ns) ProcessPendingSessionEvents:495(70599152ns) {ForwardSessionEventToSession:612(23785128ns) ForwardSessionEventToSession:612(5029618ns) ForwardSessionEventToSession:612(3933ns) ForwardSessionEventToSession:612(9945129ns) ForwardSessionEventToSession:612(6002ns) ForwardSessionEventToSession:612(2752ns) ForwardSessionEventToSession:612(1701265ns) ForwardSessionEventToSession:612(3737ns) ForwardSessionEventToSession:612(6360028ns) ForwardSessionEventToSession:612(23733728ns)}}}
2025-11-11T17:20:27.065227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518935599445378:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:27.027078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518937157872485:2107] [node 6] ICS09 handshake done sender: [3:7571518932862905169:2120] self: [3:7571518932862905168:0] peer: [6:7571518931224739399:0] socket: 448 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.036877Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32697) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:27.006839Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518931224739391:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:27.205888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518929455607828:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.027112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518937157872485:2107] [node 6] ICS10 traffic start
2025-11-11T17:20:27.027172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:27.050484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518937157872488:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47487
2025-11-11T17:20:27.058177Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518937157872484:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.058323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518928567937795:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.109750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518937157872488:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.109888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518928567937796:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.163891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518937157872491:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47487
2025-11-11T17:20:27.163946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518937157872492:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47487
2025-11-11T17:20:27.165948Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518932862905182:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.165980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518932862905182:2125] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.183564Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518932862905182:2125] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:27.186660Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518937157872484:2126] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:27.186868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518928567937795:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.186958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:27.187007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518928567937795:2048] [node 1] ICP22 created new session: [3:7571518937157872494:2048]
2025-11-11T17:20:27.187051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518937157872494:2048] [node 1] ICS09 handshake done sender: [3:7571518937157872484:2126] self: [3:7571518937157872489:0] peer: [1:7571518931582249380:0] socket: 492 qp: -1
2025-11-11T17:20:27.187084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518937157872494:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:27.187116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:27.190012Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518937157872488:2127] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:27.213941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518928567937796:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.214014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:27.214074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518928567937796:2049] [node 2] ICP22 created new session: [3:7571518937157872496:2049]
2025-11-11T17:20:27.214123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518937157872496:2049] [node 2] ICS09 handshake done sender: [3:7571518937157872488:2127] self: [3:7571518937157872490:0] peer: [2:7571518933750575203:0] socket: 455 qp: -1
2025-11-11T17:20:27.214156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518937157872496:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:27.214219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:27.214639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518932862905167:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.214717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:27.214759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518932862905167:2109] [node 5] ICP22 created new session: [3:7571518937157872498:2109]
2025-11-11T17:20:27.219985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518937157872498:2109] [node 5] ICS09 handshake done sender: [3:7571518932862905182:2125] self: [3:7571518932862905181:0] peer: [5:7571518935599445381:0] socket: 497 qp: -1
2025-11-11T17:20:27.220041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518937157872498:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:27.220104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.065261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:27.223706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518933750575206:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.026679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518935519706697:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:27.041606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518930846957818:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.258713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518931582249369:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(134519005ns) {HandleHandshakeStatus:329(127362525ns) {DropIncomingHandshake:441(637ns) DropOutgoingHandshake:456(768ns) SwitchToState:178(780ns) ProcessPendingSessionEvents:495(20186642ns) {ForwardSessionEventToSession:612(20130252ns) ForwardSessionEventToSession:612(4427ns) ForwardSessionEventToSession:612(10555ns) ForwardSessionEventToSession:612(6238ns) ForwardSessionEventToSession:612(2226ns) ForwardSessionEventToSession:612(2643ns) ForwardSessionEventToSession:612(5680ns) ForwardSessionEventToSession:612(4922ns) ForwardSessionEventToSession:612(5475ns) ForwardSessionEventToSession:612(1800ns)}}}
2025-11-11T17:20:27.258806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518931582249369:2109] [node 6] ICP18 handshake (actor [1:7571518935877216696:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:27.049098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:27.109743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518935599445375:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:27.061888Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518930846957829:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:27.063302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518930846957756:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32697) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518930846957829:2114] held: no
2025-11-11T17:20:27.122514Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518935599445376:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.066206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518930846957818:2109] [node 6] ICP22 created new session: [4:7571518935141925139:2109]
2025-11-11T17:20:27.070131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518930846957754:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(27999875ns) {HandleHandshakeStatus:395(27995741ns) {LogHandshakeFail:472(3244019ns) {UpdateErrorStateLog:544(8258ns)} DropOutgoingHandshake:456(2818ns) ProcessPendingSessionEvents:495(13024793ns) {DropSessionEvent:512(11918ns) DropSessionEvent:512(4803159ns) DropSessionEvent:512(25545ns) DropSessionEvent:512(5237ns) DropSessionEvent:512(5588ns) DropSessionEvent:512(5989ns) DropSessionEvent:512(22341ns) DropSessionEvent:512(5999ns) DropSessionEvent:512(4954ns) DropSessionEvent:512(6146ns)}}}
2025-11-11T17:20:27.122656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518931304477997:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.073624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518935141925138:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
2025-11-11T17:20:27.123176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518931304478070:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:27.073930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518930846957756:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.141967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518931304477995:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.074013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:27.223746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518933750575206:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.236945Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518938045542514:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.242201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:27.242458Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518933750575206:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:27.250346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518929455607828:2049] [node 3] ICP22 created new session: [2:7571518938045542521:2049]
2025-11-11T17:20:27.259388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518938045542521:2049] [node 3] ICS09 handshake done sender: [2:7571518933750575204:2116] self: [2:7571518933750575203:0] peer: [3:7571518937157872490:0] socket: 463 qp: -1
2025-11-11T17:20:27.265781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518938045542521:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:27.165721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:27.175091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518931304477995:2048] [node 1] ICP22 created new session: [5:7571518935599445382:2048]
2025-11-11T17:20:27.183355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518935599445382:2048] [node 1] ICS09 handshake done sender: [5:7571518931304478070:2114] self: [5:7571518931304478078:0] peer: [1:7571518931582249372:0] socket: 467 qp: -1
2025-11-11T17:20:27.186228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518935599445384:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:27.190312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518935599445382:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:27.195265Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518935599445376:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:27.206798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518935599445383:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:27.206909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518931304478075:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.206931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518931304478075:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.207654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518931304477997:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.207729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:27.207778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518931304477997:2050] [node 3] ICP22 created new session: [5:7571518935599445385:2050]
2025-11-11T17:20:27.207821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518935599445385:2050] [node 3] ICS09 handshake done sender: [5:7571518935599445376:2119] self: [5:7571518935599445381:0] peer: [3:7571518932862905181:0] socket: 450 qp: -1
2025-11-11T17:20:27.207857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518935599445385:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:27.207889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:27.209702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:27.230076Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518931304478075:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:27.254133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518931304477995:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(111943069ns) {HandleHandshakeStatus:329(111938822ns) {DropIncomingHandshake:441(2572ns) DropOutgoingHandshake:456(22438ns) SwitchToState:178(515ns) ProcessPendingSessionEvents:495(11756136ns) {ForwardSessionEventToSession:612(11674546ns) ForwardSessionEventToSession:612(7680ns) ForwardSessionEventToSession:612(6226ns) ForwardSessionEventToSession:612(3158ns) ForwardSessionEventToSession:612(2541ns) ForwardSessionEventToSession:612(10840ns) ForwardSessionEventToSession:612(3661ns) ForwardSessionEventToSession:612(23667ns) ForwardSessionEventToSession:612(6540ns) ForwardSessionEventToSession:612(2266ns)}}}
2025-11-11T17:20:27.256318Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518935599445383:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.257482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518931304478060:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.257562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:27.257623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518931304478060:2110] [node 6] ICP22 created new session: [5:7571518935599445389:2110]
2025-11-11T17:20:27.257669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518935599445389:2110] [node 6] ICS09 handshake done sender: [5:7571518931304478075:2117] self: [5:7571518931304478074:0] peer: [6:7571518935519706711:0] socket: 478 qp: -1
2025-11-11T17:20:27.257708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518935599445389:2110] [node 6] ICS10 traffic start
2025-11-11T17:20:27.257765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:27.268049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518931304478059:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.290293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518935599445392:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29254
2025-11-11T17:20:27.295874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518935599445383:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:27.037795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518931224739316:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.268239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518933750575184:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.074059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518930846957756:2050] [node 3] ICP22 created new session: [4:7571518935141925140:2050]
2025-11-11T17:20:27.303135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518931304478059:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.303201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:27.303252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518931304478059:2109] [node 4] ICP22 created new session: [5:7571518935599445393:2109]
2025-11-11T17:20:27.303321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518935599445393:2109] [node 4] ICS09 handshake done sender: [5:7571518935599445383:2121] self: [5:7571518935599445391:0] peer: [4:7571518930846957832:0] socket: 510 qp: -1
2025-11-11T17:20:27.303362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518935599445393:2109] [node 4] ICS10 traffic start
2025-11-11T17:20:27.303404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:27.273993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:27.286584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518933750575184:2107] [node 4] ICP22 created new session: [2:7571518938045542522:2107]
2025-11-11T17:20:27.286669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:27.297078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518929455607828:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(99436342ns) {HandleHandshakeStatus:329(90974653ns) {DropIncomingHandshake:441(15709ns) DropOutgoingHandshake:456(570ns) SwitchToState:178(631ns) ProcessPendingSessionEvents:495(1696417ns) {ForwardSessionEventToSession:612(1638893ns) ForwardSessionEventToSession:612(9382ns) ForwardSessionEventToSession:612(4368ns) ForwardSessionEventToSession:612(3421ns) ForwardSessionEventToSession:612(8025ns) ForwardSessionEventToSession:612(3479ns) ForwardSessionEventToSession:612(2842ns) ForwardSessionEventToSession:612(6622ns) ForwardSessionEventToSession:612(5782ns) ForwardSessionEventToSession:612(4650ns)}}}
2025-11-11T17:20:27.298749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518938045542522:2107] [node 4] ICS09 handshake done sender: [2:7571518933750575206:2117] self: [2:7571518933750575202:0] peer: [4:7571518935141925148:0] socket: 458 qp: -1
2025-11-11T17:20:27.298793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518938045542522:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:27.298841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:27.300982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518933750575184:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(32484595ns) {HandleHandshakeStatus:329(32482351ns) {DropIncomingHandshake:441(17908ns) DropOutgoingHandshake:456(540ns) SwitchToState:178(3142ns) ProcessPendingSessionEvents:495(1761284ns) {ForwardSessionEventToSession:612(1708470ns) ForwardSessionEventToSession:612(5169ns) ForwardSessionEventToSession:612(6046ns) ForwardSessionEventToSession:612(3442ns) ForwardSessionEventToSession:612(2383ns) ForwardSessionEventToSession:612(6017ns) ForwardSessionEventToSession:612(3795ns) ForwardSessionEventToSession:612(3315ns) ForwardSessionEventToSession:612(11183ns) ForwardSessionEventToSession:612(2667ns)}}}
2025-11-11T17:20:27.074110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518935141925140:2050] [node 3] ICS09 handshake done sender: [4:7571518930846957829:2114] self: [4:7571518930846957834:0] peer: [3:7571518932862905172:0] socket: 470 qp: -1
2025-11-11T17:20:27.074149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518935141925140:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:27.079463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:27.102098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518935141925139:2109] [node 6] ICS09 handshake done sender: [4:7571518930846957824:2112] self: [4:7571518930846957823:0] peer: [6:7571518931224739396:0] socket: 456 qp: -1
2025-11-11T17:20:27.110294Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518930846957830:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:27.114799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518935141925139:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:27.133800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:27.181917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518930846957754:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.181987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:27.182046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518930846957754:2048] [node 1] ICP22 created new session: [4:7571518935141925146:2048]
2025-11-11T17:20:27.182099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518935141925146:2048] [node 1] ICS09 handshake done sender: [4:7571518930846957830:2115] self: [4:7571518930846957839:0] peer: [1:7571518931582249370:0] socket: 476 qp: -1
2025-11-11T17:20:27.182138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518935141925146:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:27.182204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:27.182999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518935141925145:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
2025-11-11T17:20:27.194362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518930846957818:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(152559033ns) {HandleHandshakeStatus:329(152556136ns) {DropIncomingHandshake:441(12402ns) DropOutgoingHandshake:456(425ns) SwitchToState:178(6626ns) ProcessPendingSessionEvents:495(16197046ns) {ForwardSessionEventToSession:612(16139212ns) ForwardSessionEventToSession:612(8197ns) ForwardSessionEventToSession:612(12073ns) ForwardSessionEventToSession:612(2471ns) ForwardSessionEventToSession:612(2950ns) ForwardSessionEventToSession:612(4915ns) ForwardSessionEventToSession:612(4388ns) ForwardSessionEventToSession:612(3029ns) ForwardSessionEventToSession:612(5056ns) ForwardSessionEventToSession:612(3572ns)}}}
2025-11-11T17:20:27.194519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518935141925145:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.194886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518930846957755:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.260636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518935141925149:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10798
2025-11-11T17:20:27.278402Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518930846957835:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.278435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518930846957835:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:27.279309Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518935141925145:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:27.294907Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518930846957835:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:27.295754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518930846957755:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.295818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:27.295870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518930846957755:2049] [node 2] ICP22 created new session: [4:7571518935141925151:2049]
2025-11-11T17:20:27.295917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518935141925151:2049] [node 2] ICS09 handshake done sender: [4:7571518935141925145:2121] self: [4:7571518935141925148:0] peer: [2:7571518933750575202:0] socket: 503 qp: -1
2025-11-11T17:20:27.295949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518935141925151:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:27.295984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:27.299121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518930846957817:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:27.301304Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10677) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:27.301731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:27.305397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518930846957817:2108] [node 5] ICP22 created new session: [4:7571518935141925153:2108]
2025-11-11T17:20:27.306922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518930846957755:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:27.037847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:27.037901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518931224739316:2049] [node 2] ICP22 created new session: [6:7571518935519706699:2049]
2025-11-11T17:20:27.037947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518935519706699:2049] [node 2] ICS09 handshake done sender: [6:7571518931224739391:2115] self: [6:7571518931224739394:0] peer: [2:7571518933750575194:0] socket: 477 qp: -1
2025-11-11T17:20:27.037976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518935519706699:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:27.038004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:27.038421Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10677) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:27.045748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518935519706698:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:27.050152Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518931224739388:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:27.050790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518931224739316:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:27.050921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518931224739317:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.050965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:27.051032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518931224739317:2050] [node 3] ICP22 created new session: [6:7571518935519706701:2050]
2025-11-11T17:20:27.051081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518935519706701:2050] [node 3] ICS09 handshake done sender: [6:7571518931224739388:2113] self: [6:7571518931224739399:0] peer: [3:7571518932862905168:0] socket: 449 qp: -1
2025-11-11T17:20:27.051110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518935519706701:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:27.051142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:27.072092Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11818) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:27.092888Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518935519706698:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.104447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518931224739400:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:27.105712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518931224739367:2093] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11818) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518931224739389:2114] held: no
2025-11-11T17:20:27.106031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518931224739367:2093] [node 4] ICP35 event processing took too much time PendingConnection:278(23970267ns) {HandleHandshakeStatus:395(23967317ns) {LogHandshakeFail:472(13428ns) {UpdateErrorStateLog:544(4896ns)} DropOutgoingHandshake:456(3698ns) ProcessPendingSessionEvents:495(77911ns) {DropSessionEvent:512(14922ns) DropSessionEvent:512(10919ns) DropSessionEvent:512(4823ns) DropSessionEvent:512(6066ns) DropSessionEvent:512(3765ns) DropSessionEvent:512(5385ns) DropSessionEvent:512(5661ns) DropSessionEvent:512(3658ns) DropSessionEvent:512(5664ns) DropSessionEvent:512(4414ns)}}}
2025-11-11T17:20:27.106169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518931224739315:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.108553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518935519706707:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:27.109262Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518935519706698:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:27.110153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518935519706705:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:27.131492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518931224739315:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.135325Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518931224739389:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:27.165370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:27.175236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518931224739315:2048] [node 1] ICP22 created new session: [6:7571518935519706708:2048]
2025-11-11T17:20:27.195554Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518935519706705:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:27.195646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518931224739367:2093] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.195689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:27.195734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518931224739367:2093] [node 4] ICP22 created new session: [6:7571518935519706709:2093]
2025-11-11T17:20:27.195777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518935519706709:2093] [node 4] ICS09 handshake done sender: [6:7571518931224739389:2114] self: [6:7571518931224739396:0] peer: [4:7571518930846957823:0] socket: 466 qp: -1
2025-11-11T17:20:27.195807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518935519706709:2093] [node 4] ICS10 traffic start
2025-11-11T17:20:27.195842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:27.196453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518931224739366:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:27.205713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518935519706708:2048] [node 1] ICS09 handshake done sender: [6:7571518935519706698:2121] self: [6:7571518935519706706:0] peer: [1:7571518931582249386:0] socket: 495 qp: -1
2025-11-11T17:20:27.205762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518935519706708:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:27.223677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:27.237880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518935519706713:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17184
2025-11-11T17:20:27.257218Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518935519706705:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:27.265912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518931224739366:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:27.265968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:27.266019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518931224739366:2096] [node 5] ICP22 created new session: [6:7571518935519706714:2096]
2025-11-11T17:20:27.266062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518935519706714:2096] [node 5] ICS09 handshake done sender: [6:7571518935519706705:2122] self: [6:7571518935519706711:0] peer: [5:7571518931304478074:0] socket: 500 qp: -1
2025-11-11T17:20:27.266095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518935519706714:2096] [node 5] ICS10 traffic start
2025-11-11T17:20:27.266136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:27.294211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518931224739315:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(162453158ns) {HandleHandshakeStatus:329(162449991ns) {DropIncomingHandshake:441(14668ns) DropOutgoingHandshake:456(5289ns) SwitchToState:178(195ns) ProcessPendingSessionEvents:495(7481286ns) {ForwardSessionEventToSession:612(7422143ns) ForwardSessionEventToSession:612(11906ns) ForwardSessionEventToSession:612(5524ns) ForwardSessionEventToSession:612(5354ns) ForwardSessionEventToSession:612(3538ns) ForwardSessionEventToSession:612(2829ns) ForwardSessionEventToSession:612(5889ns) ForwardSessionEventToSession:612(3132ns) ForwardSessionEventToSession:612(2833ns) ForwardSessionEventToSession:612(5430ns)}}}
2025-11-11T17:20:27.309832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518935141925153:2108] [node 5] ICS09 handshake done sender: [4:7571518930846957835:2117] self: [4:7571518930846957832:0] peer: [5:7571518935599445391:0] socket: 508 qp: -1
2025-11-11T17:20:27.309888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518935141925153:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:27.309934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:27.405759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:27.405988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518938045542509:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-11-11T17:20:27.406038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518929455607827:2048] [node 1] ICP30 unregister session Session# [2:7571518938045542509:2048] VirtualId# [2:7571518933750575207:0]
2025-11-11T17:20:27.406072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518938045542509:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.406175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:27.406221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:27.421683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:27.422019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518937157872494:2048] [node 1] ICS01 socket: 492 reason# EndOfStream
2025-11-11T17:20:27.422063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518928567937795:2048] [node 1] ICP30 unregister session Session# [3:7571518937157872494:2048] VirtualId# [3:7571518937157872489:0]
2025-11-11T17:20:27.422101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518937157872494:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.422228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:27.422288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:27.425839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:27.426086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518935141925146:2048] [node 1] ICS01 socket: 476 reason# EndOfStream
2025-11-11T17:20:27.426126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518930846957754:2048] [node 1] ICP30 unregister session Session# [4:7571518935141925146:2048] VirtualId# [4:7571518930846957839:0]
2025-11-11T17:20:27.426161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518935141925146:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.426255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:27.426296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:27.426642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:27.426842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571518935599445382:2048] [node 1] ICS07 socket disconnect 467 reason# EndOfStream
2025-11-11T17:20:27.426870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518935599445382:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.426928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:27.426962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571518935599445382:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:27.428019Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8272) Couldn't connect to any resolved address
2025-11-11T17:20:27.428339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518931304477995:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8272) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:27.428388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518935599445382:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:27.428422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518931304477995:2048] [node 1] ICP30 unregister session Session# [5:7571518935599445382:2048] VirtualId# [5:7571518931304478078:0]
2025-11-11T17:20:27.428466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:27.428499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571518931304477995:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:27.428512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:27.433751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:27.433974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518935519706708:2048] [node 1] ICS01 socket: 495 reason# EndOfStream
2025-11-11T17:20:27.434017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518931224739315:2048] [node 1] ICP30 unregister session Session# [6:7571518935519706708:2048] VirtualId# [6:7571518935519706706:0]
2025-11-11T17:20:27.434054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518935519706708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.434146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:27.434187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:27.435338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571518931304477995:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:27.519908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:27.520187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518937157872496:2049] [node 2] ICS01 socket: 455 reason# EndOfStream
2025-11-11T17:20:27.520232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518928567937796:2049] [node 2] ICP30 unregister session Session# [3:7571518937157872496:2049] VirtualId# [3:7571518937157872490:0]
2025-11-11T17:20:27.520272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518937157872496:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.520375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:27.520424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:27.525508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:27.525748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518935599445378:2049] [node 2] ICS01 socket: 479 reason# EndOfStream
2025-11-11T17:20:27.525791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518931304477996:2049] [node 2] ICP30 unregister session Session# [5:7571518935599445378:2049] VirtualId# [5:7571518931304478076:0]
2025-11-11T17:20:27.525823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518935599445378:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.525899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:27.525960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:27.540876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:27.542439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518935141925151:2049] [node 2] ICS01 socket: 503 reason# EndOfStream
2025-11-11T17:20:27.542482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518930846957755:2049] [node 2] ICP30 unregister session Session# [4:7571518935141925151:2049] VirtualId# [4:7571518935141925148:0]
2025-11-11T17:20:27.541739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:27.541966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518935519706699:2049] [node 2] ICS01 socket: 477 reason# EndOfStream
2025-11-11T17:20:27.542012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518931224739316:2049] [node 2] ICP30 unregister session Session# [6:7571518935519706699:2049] VirtualId# [6:7571518931224739394:0]
2025-11-11T17:20:27.542053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518935519706699:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.542171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:27.542220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:27.542790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518935141925151:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.542864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:27.542901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:27.673773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:27.674004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518935519706701:2050] [node 3] ICS01 socket: 449 reason# EndOfStream
2025-11-11T17:20:27.674050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518931224739317:2050] [node 3] ICP30 unregister session Session# [6:7571518935519706701:2050] VirtualId# [6:7571518931224739399:0]
2025-11-11T17:20:27.674089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518935519706701:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.674179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:27.674242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:27.674353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:27.674529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518935141925140:2050] [node 3] ICS01 socket: 470 reason# EndOfStream
2025-11-11T17:20:27.674567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518930846957756:2050] [node 3] ICP30 unregister session Session# [4:7571518935141925140:2050] VirtualId# [4:7571518930846957834:0]
2025-11-11T17:20:27.674596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518935141925140:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.674661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:27.674718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:27.673775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:27.674941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518935599445385:2050] [node 3] ICS01 socket: 450 reason# EndOfStream
2025-11-11T17:20:27.674970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518931304477997:2050] [node 3] ICP30 unregister session Session# [5:7571518935599445385:2050] VirtualId# [5:7571518935599445381:0]
2025-11-11T17:20:27.674996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518935599445385:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.675050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:27.675091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:27.833774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:27.833856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:27.834006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518935519706709:2093] [node 4] ICS01 socket: 466 reason# EndOfStream
2025-11-11T17:20:27.834047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518931224739367:2093] [node 4] ICP30 unregister session Session# [6:7571518935519706709:2093] VirtualId# [6:7571518931224739396:0]
2025-11-11T17:20:27.834092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518935599445393:2109] [node 4] ICS01 socket: 510 reason# EndOfStream
2025-11-11T17:20:27.834127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518931304478059:2109] [node 4] ICP30 unregister session Session# [5:7571518935599445393:2109] VirtualId# [5:7571518935599445391:0]
2025-11-11T17:20:27.834081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518935519706709:2093] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.834170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:27.834156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518935599445393:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.834211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:27.834253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:27.834299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:27.965769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:27.965985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518935519706714:2096] [node 5] ICS01 socket: 500 reason# EndOfStream
2025-11-11T17:20:27.966028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518931224739366:2096] [node 5] ICP30 unregister session Session# [6:7571518935519706714:2096] VirtualId# [6:7571518935519706711:0]
2025-11-11T17:20:27.966062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518935519706714:2096] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:27.966171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:27.966218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:28.153990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518940380397877:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:28.154028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518940380397878:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:28.177689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518941219798987:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:28.179346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518941219798986:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:28.206182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518942170622958:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:28.206222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518942170622959:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:28.229191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518939784726686:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:28.229233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518939784726687:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:28.229268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518939784726688:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:28.267744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518941368218031:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:28.267781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518941368218032:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:28.267933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518941368218030:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:28.294559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518941382990327:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:28.294592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518941382990328:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:28.295097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518941382990326:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.551397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518941368218050:2064] [node 4] ICP01 ready to work
2025-11-11T17:20:28.642728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518939784726686:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(131121429ns) {Configure:104(131117106ns) {ProcessConfigured:135(55463465ns) {StartInitialHandshake:157(55460530ns) {DropHandshakes:469(2532ns) {DropIncomingHandshake:441(528ns) DropOutgoingHandshake:456(586ns)} GenerateSessionVirtualId:988(711ns) PrepareNewSessionHandshake:476(4824ns) {SwitchToState:178(1776ns)}}}}}
2025-11-11T17:20:28.569701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518942170622970:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17905
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.644954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518939784726709:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.762555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518941382990352:2072] [node 4] ICP01 ready to work
2025-11-11T17:20:28.771921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518941219799050:2065] [node 4] ICP01 ready to work
2025-11-11T17:20:28.692168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518940380397877:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(213572000ns) {Configure:104(213567165ns) {ProcessConfigured:135(26152433ns) {StartInitialHandshake:157(26148272ns) {DropHandshakes:469(3008ns) {DropIncomingHandshake:441(1039ns) DropOutgoingHandshake:456(477ns)} GenerateSessionVirtualId:988(1156ns) PrepareNewSessionHandshake:476(5822ns) {SwitchToState:178(2107ns)}}}}}
2025-11-11T17:20:28.794296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518941368218101:2060] [node 6] ICP01 ready to work
2025-11-11T17:20:28.724885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518939784726737:2062] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.692721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518940380397943:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51761
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.801737Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518942170622970:2060] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.788953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518941219799049:2062] [node 5] ICP01 ready to work
2025-11-11T17:20:28.803070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518941368218103:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:28.789705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518939784726742:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:28.803918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518941382990389:2061] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.769756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518940380397945:2105] [node 5] ICP01 ready to work
2025-11-11T17:20:28.807103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518941382990352:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(44252140ns) {Configure:104(44245793ns) {ProcessConfigured:135(72384ns) {StartInitialHandshake:157(69395ns) {DropHandshakes:469(2624ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(545ns)} GenerateSessionVirtualId:988(983ns) PrepareNewSessionHandshake:476(2116ns) {SwitchToState:178(1121ns)}}}}}
2025-11-11T17:20:28.821193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518941219799054:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15158
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.815591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518941368218101:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20941173ns) {Configure:104(20936040ns) {ProcessConfigured:135(92226ns) {StartInitialHandshake:157(89455ns) {DropHandshakes:469(2847ns) {DropIncomingHandshake:441(637ns) DropOutgoingHandshake:456(373ns)} GenerateSessionVirtualId:988(1098ns) PrepareNewSessionHandshake:476(2234ns) {SwitchToState:178(501ns)}}}}}
2025-11-11T17:20:28.805689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518940380397949:2106] [node 6] ICP01 ready to work
2025-11-11T17:20:28.875027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518941219799055:2070] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.836223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518942170622958:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(209288189ns) {Configure:104(209279404ns) {ProcessConfigured:135(156180ns) {StartInitialHandshake:157(153680ns) {DropHandshakes:469(2769ns) {DropIncomingHandshake:441(587ns) DropOutgoingHandshake:456(375ns)} GenerateSessionVirtualId:988(900ns) PrepareNewSessionHandshake:476(2053ns) {SwitchToState:178(587ns)}}}}}
2025-11-11T17:20:28.855063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518941368218030:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(292908920ns) {Configure:104(292904258ns) {ProcessConfigured:135(79871ns) {StartInitialHandshake:157(76850ns) {DropHandshakes:469(2525ns) {DropIncomingHandshake:441(686ns) DropOutgoingHandshake:456(378ns)} GenerateSessionVirtualId:988(1167ns) PrepareNewSessionHandshake:476(1734ns) {SwitchToState:178(569ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.819159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518941382990326:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(345059665ns) {Configure:104(325303870ns) {ProcessConfigured:135(56246ns) {StartInitialHandshake:157(51968ns) {DropHandshakes:469(2875ns) {DropIncomingHandshake:441(606ns) DropOutgoingHandshake:456(577ns)} GenerateSessionVirtualId:988(1047ns) PrepareNewSessionHandshake:476(1622ns) {SwitchToState:178(628ns)}}}}}
2025-11-11T17:20:28.853706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518940380397950:2103] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.874741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518942170623025:2062] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.910415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518939784726709:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(265122904ns) {Configure:104(265118396ns) {ProcessConfigured:135(16474465ns) {StartInitialHandshake:157(16470962ns) {DropHandshakes:469(2491ns) {DropIncomingHandshake:441(685ns) DropOutgoingHandshake:456(304ns)} GenerateSessionVirtualId:988(623ns) PrepareNewSessionHandshake:476(5148ns) {SwitchToState:178(2653ns)}}}}}
2025-11-11T17:20:28.912453Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518941368218103:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.879130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518941219798986:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(320980071ns) {Configure:104(320963033ns) {ProcessConfigured:135(56026ns) {StartInitialHandshake:157(53002ns) {DropHandshakes:469(3068ns) {DropIncomingHandshake:441(549ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:476(1931ns) {SwitchToState:178(904ns)}}}}}
2025-11-11T17:20:28.819618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518941382990396:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:28.848854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518941382990389:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25140347ns) {Configure:104(25134256ns) {ProcessConfigured:135(109693ns) {StartInitialHandshake:157(106991ns) {DropHandshakes:469(5269ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(502ns)} GenerateSessionVirtualId:988(1059ns) PrepareNewSessionHandshake:476(2479ns) {SwitchToState:178(897ns)}}}}}
2025-11-11T17:20:28.870925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518941382990327:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(324969203ns) {Configure:104(324964234ns) {ProcessConfigured:135(68139ns) {StartInitialHandshake:157(65267ns) {DropHandshakes:469(2689ns) {DropIncomingHandshake:441(610ns) DropOutgoingHandshake:456(532ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:476(1652ns) {SwitchToState:178(852ns)}}}}}
2025-11-11T17:20:28.874439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518941382990403:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:28.877673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518941382990396:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.893731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518941382990389:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:28.918561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518941382990405:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:28.946140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518941382990406:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:28.955392Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518941382990396:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.910591Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518939784726742:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.865757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518940380397951:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51761
2025-11-11T17:20:28.918190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518942170623026:2078] [node 5] ICP01 ready to work
2025-11-11T17:20:28.917030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518941219799054:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.922249Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518941368218105:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.964539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518942170623028:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17905
2025-11-11T17:20:28.990152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518941382990389:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:28.911187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518939784726755:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:28.865961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518940380397952:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51761
2025-11-11T17:20:28.929794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518941219798986:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.922286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518941368218105:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:28.997932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:28.965798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518942170623027:2080] [node 6] ICP01 ready to work
2025-11-11T17:20:28.921661Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518940380397943:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.914668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518939784726688:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(436530722ns) {Configure:104(436526380ns) {ProcessConfigured:135(51177ns) {StartInitialHandshake:157(45377ns) {DropHandshakes:469(2503ns) {DropIncomingHandshake:441(575ns) DropOutgoingHandshake:456(481ns)} GenerateSessionVirtualId:988(465ns) PrepareNewSessionHandshake:476(1560ns) {SwitchToState:178(640ns)}}}}}
2025-11-11T17:20:28.962082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518939784726755:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.981706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518939784726757:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.981762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518939784726757:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.021958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518939784726737:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(296609097ns) {Configure:104(296601274ns) {ProcessConfigured:135(39980043ns) {StartInitialHandshake:157(39976755ns) {DropHandshakes:469(2897ns) {DropIncomingHandshake:441(666ns) DropOutgoingHandshake:456(563ns)} GenerateSessionVirtualId:988(715ns) PrepareNewSessionHandshake:476(3915ns) {SwitchToState:178(2429ns)}}}}}
2025-11-11T17:20:29.026415Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518939784726757:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:29.057436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518939784726760:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:28.932739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518941219799063:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15158
2025-11-11T17:20:28.923191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518941368218105:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:28.923478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518941368218101:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:28.923542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:28.923589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518941368218101:2060] [node 6] ICP22 created new session: [5:7571518941368218111:2060]
2025-11-11T17:20:28.923633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518941368218111:2060] [node 6] ICS09 handshake done sender: [5:7571518941368218105:2114] self: [5:7571518941368218104:0] peer: [6:7571518941382990404:0] socket: 461 qp: -1
2025-11-11T17:20:28.923672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518941368218111:2060] [node 6] ICS10 traffic start
2025-11-11T17:20:28.923707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:28.933981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518941368218115:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:28.937717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518941368218030:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:28.942424Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518941368218115:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.942569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518941368218050:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.001966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518941368218118:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:29.014714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518941368218117:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:29.022896Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518941368218103:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:29.038369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518945663185415:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:29.041794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518941368218030:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.075434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518941368218109:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:29.076196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:28.999097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518942170623028:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.921750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518940380397951:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.102065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518941368218030:2048] [node 1] ICP22 created new session: [5:7571518945663185416:2048]
2025-11-11T17:20:28.997998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518941382990389:2061] [node 5] ICP22 created new session: [6:7571518941382990407:2061]
2025-11-11T17:20:28.954180Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518941219799063:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.111012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518941368218050:2064] [node 4] ICP35 event processing took too much time PendingConnection:278(23785439ns)
2025-11-11T17:20:29.122312Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518941368218115:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:29.133699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518945663185416:2048] [node 1] ICS09 handshake done sender: [5:7571518941368218103:2113] self: [5:7571518941368218114:0] peer: [1:7571518940380397946:0] socket: 459 qp: -1
2025-11-11T17:20:28.934335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518940380397956:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51761
2025-11-11T17:20:29.006169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518942170622959:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(500121388ns) {Configure:104(487691380ns) {ProcessConfigured:135(48078910ns) {StartInitialHandshake:157(40232541ns) {DropHandshakes:469(3119ns) {DropIncomingHandshake:441(209ns) DropOutgoingHandshake:456(486ns)} GenerateSessionVirtualId:988(2825ns) PrepareNewSessionHandshake:476(3973ns) {SwitchToState:178(417ns)}}}}}
2025-11-11T17:20:28.998050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518941382990407:2061] [node 5] ICS09 handshake done sender: [6:7571518941382990396:2112] self: [6:7571518941382990404:0] peer: [5:7571518941368218104:0] socket: 462 qp: -1
2025-11-11T17:20:29.113138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518939784726709:2067] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.068391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518946465590332:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17905
2025-11-11T17:20:28.974202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518941219798987:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(459935902ns) {Configure:104(459932175ns) {ProcessConfigured:135(7562984ns) {StartInitialHandshake:157(7560127ns) {DropHandshakes:469(2346ns) {DropIncomingHandshake:441(638ns) DropOutgoingHandshake:456(414ns)} GenerateSessionVirtualId:988(1089ns) PrepareNewSessionHandshake:476(3106ns) {SwitchToState:178(296ns)}}}}}
2025-11-11T17:20:29.143284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518945663185416:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:29.113724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518939784726760:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.946671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518940380397947:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.160079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:28.998089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518941382990407:2061] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.946704Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518940380397947:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.107743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518946465590333:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17905
2025-11-11T17:20:28.998130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:28.998886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518941382990405:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.022370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518941382990327:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.029690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518941382990403:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.062696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518941382990326:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.163426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518945677957711:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:29.163581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518945677957712:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:29.188098Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518941382990403:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:29.188362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518941382990326:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.188464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:29.188519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518941382990326:2048] [node 1] ICP22 created new session: [6:7571518945677957713:2048]
2025-11-11T17:20:29.188573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518945677957713:2048] [node 1] ICS09 handshake done sender: [6:7571518941382990403:2115] self: [6:7571518945677957706:0] peer: [1:7571518940380397953:0] socket: 473 qp: -1
2025-11-11T17:20:29.188614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518945677957713:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:29.188649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:29.190160Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518945677957711:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.195249Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518941382990328:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(364037958ns) {Configure:104(364032106ns) {ProcessConfigured:135(47821762ns) {StartInitialHandshake:157(47819263ns) {DropHandshakes:469(2680ns) {DropIncomingHandshake:441(626ns) DropOutgoingHandshake:456(462ns)} GenerateSessionVirtualId:988(657ns) PrepareNewSessionHandshake:476(4228ns) {SwitchToState:178(1706ns)}}}}}
2025-11-11T17:20:29.223163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518941382990352:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:28.974625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518941219799066:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15158
2025-11-11T17:20:29.125644Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518939784726687:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(468427938ns) {Configure:104(468422588ns) {ProcessConfigured:135(52403867ns) {StartInitialHandshake:157(52400648ns) {DropHandshakes:469(2350ns) {DropIncomingHandshake:441(474ns) DropOutgoingHandshake:456(346ns)} GenerateSessionVirtualId:988(908ns) PrepareNewSessionHandshake:476(3895ns) {SwitchToState:178(1712ns)}}}}}
2025-11-11T17:20:29.127850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518942170623025:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(215598329ns) {Configure:104(210811758ns) {ProcessConfigured:135(126191ns) {StartInitialHandshake:157(123143ns) {DropHandshakes:469(2691ns) {DropIncomingHandshake:441(563ns) DropOutgoingHandshake:456(549ns)} GenerateSessionVirtualId:988(989ns) PrepareNewSessionHandshake:476(1805ns) {SwitchToState:178(745ns)}}}}}
2025-11-11T17:20:29.160674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518941368218030:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(118667200ns) {HandleHandshakeStatus:329(118664277ns) {DropIncomingHandshake:441(970ns) DropOutgoingHandshake:456(6381ns) SwitchToState:178(512ns) ProcessPendingSessionEvents:495(155795ns) {ForwardSessionEventToSession:612(80736ns) ForwardSessionEventToSession:612(10980ns) ForwardSessionEventToSession:612(17635ns) ForwardSessionEventToSession:612(6811ns) ForwardSessionEventToSession:612(3283ns) ForwardSessionEventToSession:612(3859ns) ForwardSessionEventToSession:612(6009ns) ForwardSessionEventToSession:612(3674ns) ForwardSessionEventToSession:612(6971ns) ForwardSessionEventToSession:612(6339ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.947428Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518940380397956:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.975191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518941219799054:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:28.975416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518941219798986:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:28.975494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:28.975562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518941219798986:2048] [node 1] ICP22 created new session: [2:7571518941219799067:2048]
2025-11-11T17:20:28.975605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518941219799067:2048] [node 1] ICS09 handshake done sender: [2:7571518941219799054:2111] self: [2:7571518941219799062:0] peer: [1:7571518940380397940:0] socket: 464 qp: -1
2025-11-11T17:20:28.975643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518941219799067:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:28.975701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:29.053842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518941219799060:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.053896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518941219799060:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.062168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518945514766366:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15158
2025-11-11T17:20:29.102067Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518941219799060:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:29.102130Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518945514766366:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.148553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518941219799055:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.162008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518941219799052:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.162041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518941219799052:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.166881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518941219799052:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:29.181970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:29.195677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518941219799049:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(364109990ns) {Configure:104(364100822ns) {ProcessConfigured:135(23954703ns) {StartInitialHandshake:157(23952444ns) {DropHandshakes:469(2546ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(439ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:476(4003ns) {SwitchToState:178(1702ns)}}}}}
2025-11-11T17:20:29.195988Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518941219799065:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.196011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518941219799065:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.196777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518941219799050:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.196873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:29.196927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518941219799050:2065] [node 4] ICP22 created new session: [2:7571518945514766373:2065]
2025-11-11T17:20:29.196983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518945514766373:2065] [node 4] ICS09 handshake done sender: [2:7571518941219799052:2110] self: [2:7571518941219799051:0] peer: [4:7571518944079694067:0] socket: 457 qp: -1
2025-11-11T17:20:29.197017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518945514766373:2065] [node 4] ICS10 traffic start
2025-11-11T17:20:29.197058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:29.209750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518941219799055:2070] [node 6] ICP22 created new session: [2:7571518945514766372:2070]
2025-11-11T17:20:29.210247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518945514766377:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15158
2025-11-11T17:20:29.214218Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518941219799065:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:29.214563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518941219798987:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.214705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:29.214764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518941219798987:2049] [node 3] ICP22 created new session: [2:7571518945514766379:2049]
2025-11-11T17:20:29.214815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518945514766379:2049] [node 3] ICS09 handshake done sender: [2:7571518941219799065:2115] self: [2:7571518941219799064:0] peer: [3:7571518946465590337:0] socket: 495 qp: -1
2025-11-11T17:20:29.214852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518945514766379:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:29.214911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:29.224198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518945514766372:2070] [node 6] ICS09 handshake done sender: [2:7571518941219799060:2113] self: [2:7571518941219799059:0] peer: [6:7571518945677957705:0] socket: 476 qp: -1
2025-11-11T17:20:29.237780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518945514766372:2070] [node 6] ICS10 traffic start
2025-11-11T17:20:29.241815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:29.250179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518945514766377:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.160752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518941368218050:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.157669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518939784726687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.166732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518946465590332:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:28.947553Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518940380397942:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.270208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518941219799055:2070] [node 6] ICP35 event processing took too much time PendingConnection:278(121395619ns) {HandleHandshakeStatus:329(121392317ns) {DropIncomingHandshake:441(965ns) DropOutgoingHandshake:456(587ns) SwitchToState:178(527ns) ProcessPendingSessionEvents:495(136123ns) {ForwardSessionEventToSession:612(78532ns) ForwardSessionEventToSession:612(8102ns) ForwardSessionEventToSession:612(11297ns) ForwardSessionEventToSession:612(3261ns) ForwardSessionEventToSession:612(6272ns) ForwardSessionEventToSession:612(2909ns) ForwardSessionEventToSession:612(3371ns) ForwardSessionEventToSession:612(4560ns) ForwardSessionEventToSession:612(2167ns) ForwardSessionEventToSession:612(5063ns)}}}
2025-11-11T17:20:29.270392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518941219799055:2070] [node 6] ICP18 handshake (actor [2:7571518945514766377:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.165907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:29.174774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518942170623030:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:28.947572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518940380397942:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.160862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:29.263652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518945677957717:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.174802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518942170623030:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.299011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518945677957710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:29.302152Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518945677957711:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:29.302530Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13883) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.306065Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.177771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518939784726709:2067] [node 5] ICP22 created new session: [4:7571518944079694068:2067]
2025-11-11T17:20:28.954716Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518940380397947:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.160928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518941368218050:2064] [node 4] ICP22 created new session: [5:7571518945663185418:2064]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.189921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518944079694068:2067] [node 5] ICS09 handshake done sender: [4:7571518939784726757:2113] self: [4:7571518939784726756:0] peer: [5:7571518941368218116:0] socket: 478 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.175607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518942170622959:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:28.958145Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518940380397942:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:29.002372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518940380397878:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(432065039ns) {Configure:104(432056764ns) {ProcessConfigured:135(15893138ns) {StartInitialHandshake:157(15890804ns) {DropHandshakes:469(2622ns) {DropIncomingHandshake:441(554ns) DropOutgoingHandshake:456(340ns)} GenerateSessionVirtualId:988(411ns) PrepareNewSessionHandshake:476(4029ns) {SwitchToState:178(2179ns)}}}}}
2025-11-11T17:20:29.057781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518940380397952:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.078957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518940380397877:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.079085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:29.079137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518940380397877:2048] [node 2] ICP22 created new session: [1:7571518944675365260:2048]
2025-11-11T17:20:29.079221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518944675365260:2048] [node 2] ICS09 handshake done sender: [1:7571518940380397942:2111] self: [1:7571518940380397940:0] peer: [2:7571518941219799062:0] socket: 456 qp: -1
2025-11-11T17:20:29.079264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518944675365260:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:29.079329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:29.082106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518940380397945:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.126681Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518940380397954:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.126734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518940380397954:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.145826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:29.148138Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518940380397954:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:29.157875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518940380397945:2105] [node 5] ICP22 created new session: [1:7571518944675365264:2105]
2025-11-11T17:20:29.166007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518944675365264:2105] [node 5] ICS09 handshake done sender: [1:7571518940380397947:2113] self: [1:7571518940380397946:0] peer: [5:7571518941368218114:0] socket: 458 qp: -1
2025-11-11T17:20:29.166054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518944675365264:2105] [node 5] ICS10 traffic start
2025-11-11T17:20:29.167186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:29.178341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518940380397950:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(300119276ns) {Configure:104(300111448ns) {ProcessConfigured:135(107557ns) {StartInitialHandshake:157(104533ns) {DropHandshakes:469(12376ns) {DropIncomingHandshake:441(571ns) DropOutgoingHandshake:456(510ns)} GenerateSessionVirtualId:988(1217ns) PrepareNewSessionHandshake:476(1548ns) {SwitchToState:178(429ns)}}}}}
2025-11-11T17:20:29.197805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518940380397949:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.197918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:29.197970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518940380397949:2106] [node 6] ICP22 created new session: [1:7571518944675365270:2106]
2025-11-11T17:20:29.198038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518944675365270:2106] [node 6] ICS09 handshake done sender: [1:7571518940380397954:2116] self: [1:7571518940380397953:0] peer: [6:7571518945677957706:0] socket: 472 qp: -1
2025-11-11T17:20:29.198072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518944675365270:2106] [node 6] ICS10 traffic start
2025-11-11T17:20:29.198105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:29.242310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518940380397945:2105] [node 5] ICP35 event processing took too much time PendingConnection:278(167869892ns) {HandleHandshakeStatus:329(167866983ns) {DropIncomingHandshake:441(1804ns) DropOutgoingHandshake:456(1040ns) SwitchToState:178(913ns) ProcessPendingSessionEvents:495(8623937ns) {ForwardSessionEventToSession:612(4633823ns) ForwardSessionEventToSession:612(3933521ns) ForwardSessionEventToSession:612(3156ns) ForwardSessionEventToSession:612(5530ns) ForwardSessionEventToSession:612(3324ns) ForwardSessionEventToSession:612(2446ns) ForwardSessionEventToSession:612(9874ns) ForwardSessionEventToSession:612(2903ns) ForwardSessionEventToSession:612(2453ns) ForwardSessionEventToSession:612(8488ns)}}}
2025-11-11T17:20:29.246056Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518944675365258:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.246096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518944675365258:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.264384Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518944675365258:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:29.278476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518940380397878:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.313771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:29.322013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518940380397878:2049] [node 3] ICP22 created new session: [1:7571518944675365273:2049]
2025-11-11T17:20:29.361701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518944675365273:2049] [node 3] ICS09 handshake done sender: [1:7571518944675365258:2118] self: [1:7571518940380397961:0] peer: [3:7571518946465590339:0] socket: 498 qp: -1
2025-11-11T17:20:29.361763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518944675365273:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:29.160989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518945663185418:2064] [node 4] ICS09 handshake done sender: [5:7571518941368218115:2117] self: [5:7571518941368218116:0] peer: [4:7571518939784726756:0] socket: 479 qp: -1
2025-11-11T17:20:29.313829Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518945514766371:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.217877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518944079694068:2067] [node 5] ICS10 traffic start
2025-11-11T17:20:29.233316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:29.237922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518944079694063:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.237950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518944079694063:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.254027Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518944079694063:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:29.277839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518939784726737:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.277961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:29.278020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518939784726737:2062] [node 6] ICP22 created new session: [4:7571518944079694073:2062]
2025-11-11T17:20:29.278064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518944079694073:2062] [node 6] ICS09 handshake done sender: [4:7571518944079694063:2116] self: [4:7571518939784726764:0] peer: [6:7571518945677957715:0] socket: 500 qp: -1
2025-11-11T17:20:29.286970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518944079694073:2062] [node 6] ICS10 traffic start
2025-11-11T17:20:29.306304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:29.312234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518939784726709:2067] [node 5] ICP35 event processing took too much time PendingConnection:278(205616763ns) {HandleHandshakeStatus:329(205613444ns) {DropIncomingHandshake:441(19742ns) DropOutgoingHandshake:456(421ns) SwitchToState:178(729ns) ProcessPendingSessionEvents:495(6010639ns) {ForwardSessionEventToSession:612(5951683ns) ForwardSessionEventToSession:612(9600ns) ForwardSessionEventToSession:612(3469ns) ForwardSessionEventToSession:612(7774ns) ForwardSessionEventToSession:612(6419ns) ForwardSessionEventToSession:612(2528ns) ForwardSessionEventToSession:612(5739ns) ForwardSessionEventToSession:612(3461ns) ForwardSessionEventToSession:612(3053ns) ForwardSessionEventToSession:612(4872ns)}}}
2025-11-11T17:20:29.313399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518944079694075:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:29.313797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518944079694069:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:29.374098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518944079694075:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.380162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518944079694070:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:29.381861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518939784726688:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.326143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518941382990327:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13883) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518941382990405:2116] held: no
2025-11-11T17:20:29.182167Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518946465590333:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.182413Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518942170623030:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:29.190061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518942170622958:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.234991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518942170623027:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(248233929ns) {Configure:104(241880095ns) {ProcessConfigured:135(76243ns) {StartInitialHandshake:157(73620ns) {DropHandshakes:469(6077ns) {DropIncomingHandshake:441(430ns) DropOutgoingHandshake:456(771ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:476(2100ns) {SwitchToState:178(512ns)}}}}}
2025-11-11T17:20:29.245910Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13883) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.254470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518942170623026:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.257755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518942170622959:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13883) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518946465590332:2115] held: no
2025-11-11T17:20:29.266343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518942170622959:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19825423ns) {HandleHandshakeStatus:395(19822386ns) {LogHandshakeFail:472(17342ns) {UpdateErrorStateLog:544(7005ns)} DropOutgoingHandshake:456(2368ns) ProcessPendingSessionEvents:495(8307719ns) {DropSessionEvent:512(13230ns) DropSessionEvent:512(6859ns) DropSessionEvent:512(8234766ns) DropSessionEvent:512(14597ns) DropSessionEvent:512(5674ns) DropSessionEvent:512(4463ns) DropSessionEvent:512(5201ns) DropSessionEvent:512(5403ns) DropSessionEvent:512(5965ns) DropSessionEvent:512(4718ns)}}}
2025-11-11T17:20:29.297423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518946465590342:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17905
2025-11-11T17:20:29.300805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:29.310041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518946465590343:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17905
2025-11-11T17:20:29.311508Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518946465590332:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:29.317663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518942170623026:2078] [node 5] ICP22 created new session: [3:7571518946465590347:2078]
2025-11-11T17:20:29.322050Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12746) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.332499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518942170622959:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.332611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:29.332671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518942170622959:2049] [node 2] ICP22 created new session: [3:7571518946465590348:2049]
2025-11-11T17:20:29.332712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518946465590348:2049] [node 2] ICS09 handshake done sender: [3:7571518946465590332:2115] self: [3:7571518946465590337:0] peer: [2:7571518941219799064:0] socket: 496 qp: -1
2025-11-11T17:20:29.332744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518946465590348:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:29.332781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:29.333245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518942170622958:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12746) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518946465590333:2116] held: no
2025-11-11T17:20:29.337750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518946465590347:2078] [node 5] ICS09 handshake done sender: [3:7571518942170623030:2113] self: [3:7571518942170623029:0] peer: [5:7571518945663185420:0] socket: 490 qp: -1
2025-11-11T17:20:29.337863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518946465590344:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17905
2025-11-11T17:20:29.350821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518946465590343:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.353739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518946465590347:2078] [node 5] ICS10 traffic start
2025-11-11T17:20:29.353836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:29.354351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518942170623026:2078] [node 5] ICP35 event processing took too much time PendingConnection:278(99650685ns) {HandleHandshakeStatus:329(99646967ns) {DropIncomingHandshake:441(16347ns) DropOutgoingHandshake:456(567ns) SwitchToState:178(3806ns) ProcessPendingSessionEvents:495(115533ns) {ForwardSessionEventToSession:612(61744ns) ForwardSessionEventToSession:612(7634ns) ForwardSessionEventToSession:612(5763ns) ForwardSessionEventToSession:612(3020ns) ForwardSessionEventToSession:612(2926ns) ForwardSessionEventToSession:612(7910ns) ForwardSessionEventToSession:612(4098ns) ForwardSessionEventToSession:612(2966ns) ForwardSessionEventToSession:612(5274ns) ForwardSessionEventToSession:612(5122ns)}}}
2025-11-11T17:20:29.379252Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518946465590333:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:29.394064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518942170622958:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.394168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:29.394223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518942170622958:2048] [node 1] ICP22 created new session: [3:7571518946465590352:2048]
2025-11-11T17:20:29.394277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518946465590352:2048] [node 1] ICS09 handshake done sender: [3:7571518946465590333:2116] self: [3:7571518946465590339:0] peer: [1:7571518940380397961:0] socket: 499 qp: -1
2025-11-11T17:20:29.394320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518946465590352:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:29.398261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:29.398155Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518939784726755:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:29.313881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518945514766371:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.161030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518945663185418:2064] [node 4] ICS10 traffic start
2025-11-11T17:20:29.326318Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518941382990352:2072] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518945677957711:2120] held: no
2025-11-11T17:20:29.340724Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518945514766371:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:29.161090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:29.409736Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518944079694070:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.394741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:29.326418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518941382990352:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.433499Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518946465590335:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.433559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518946465590335:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.440083Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518946465590341:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.440108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518946465590341:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.441214Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518946465590341:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:29.441452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518942170623027:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.441551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:29.441618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518942170623027:2080] [node 6] ICP22 created new session: [3:7571518946465590355:2080]
2025-11-11T17:20:29.449772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518946465590355:2080] [node 6] ICS09 handshake done sender: [3:7571518946465590341:2118] self: [3:7571518946465590340:0] peer: [6:7571518945677957723:0] socket: 468 qp: -1
2025-11-11T17:20:29.449820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518946465590355:2080] [node 6] ICS10 traffic start
2025-11-11T17:20:29.449892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:29.450532Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518946465590335:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:29.165718Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518941368218118:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.410601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518939784726687:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.441918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518944675365267:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.346580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518945514766383:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15158
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.326492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:29.441969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518944675365267:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:29.459834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518940380397878:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(181011920ns) {HandleHandshakeStatus:329(181009314ns) {DropIncomingHandshake:441(18995ns) DropOutgoingHandshake:456(207ns) SwitchToState:178(682ns) ProcessPendingSessionEvents:495(9905220ns) {ForwardSessionEventToSession:612(9846610ns) ForwardSessionEventToSession:612(15896ns) ForwardSessionEventToSession:612(3116ns) ForwardSessionEventToSession:612(2617ns) ForwardSessionEventToSession:612(4834ns) ForwardSessionEventToSession:612(5376ns) ForwardSessionEventToSession:612(2426ns) ForwardSessionEventToSession:612(5306ns) ForwardSessionEventToSession:612(2683ns) ForwardSessionEventToSession:612(5901ns)}}}
2025-11-11T17:20:29.465873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518942170622958:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(56701720ns) {HandleHandshakeStatus:329(56700439ns) {DropIncomingHandshake:441(330ns) DropOutgoingHandshake:456(203ns) SwitchToState:178(166ns) ProcessPendingSessionEvents:495(988ns)}}
2025-11-11T17:20:29.169793Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:29.393748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518941219799049:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.413472Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:29.413741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518941219799050:2065] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:29.426024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:29.433801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518941219799049:2062] [node 5] ICP22 created new session: [2:7571518945514766384:2062]
2025-11-11T17:20:29.449511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518945514766384:2062] [node 5] ICS09 handshake done sender: [2:7571518945514766371:2118] self: [2:7571518945514766369:0] peer: [5:7571518945663185423:0] socket: 466 qp: -1
2025-11-11T17:20:29.450684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518945514766384:2062] [node 5] ICS10 traffic start
2025-11-11T17:20:29.454741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.326568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518941382990352:2072] [node 4] ICP22 created new session: [6:7571518945677957719:2072]
2025-11-11T17:20:29.485674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518942170623025:2062] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.410702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:29.172338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518941368218101:2060] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:29.410756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518939784726687:2049] [node 2] ICP22 created new session: [4:7571518944079694078:2049]
2025-11-11T17:20:29.410801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518944079694078:2049] [node 2] ICS09 handshake done sender: [4:7571518939784726755:2112] self: [4:7571518944079694067:0] peer: [2:7571518941219799051:0] socket: 465 qp: -1
2025-11-11T17:20:29.410838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518944079694078:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:29.410890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:29.425796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518939784726686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.457966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518939784726737:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(136035983ns) {HandleHandshakeStatus:329(136033154ns) {DropIncomingHandshake:441(13944ns) DropOutgoingHandshake:456(452ns) SwitchToState:178(583ns) ProcessPendingSessionEvents:495(33706745ns) {ForwardSessionEventToSession:612(14163113ns) ForwardSessionEventToSession:612(6484ns) ForwardSessionEventToSession:612(4519ns) ForwardSessionEventToSession:612(10390ns) ForwardSessionEventToSession:612(3364ns) ForwardSessionEventToSession:612(838400ns) ForwardSessionEventToSession:612(4840186ns) ForwardSessionEventToSession:612(8673ns) ForwardSessionEventToSession:612(13805926ns) ForwardSessionEventToSession:612(4309ns)}}}
2025-11-11T17:20:29.462080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518944079694081:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:29.478395Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518944079694075:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:29.485787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518939784726688:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.485911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:29.485989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518939784726688:2050] [node 3] ICP22 created new session: [4:7571518944079694082:2050]
2025-11-11T17:20:29.486038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518944079694082:2050] [node 3] ICS09 handshake done sender: [4:7571518944079694075:2120] self: [4:7571518944079694077:0] peer: [3:7571518946465590334:0] socket: 471 qp: -1
2025-11-11T17:20:29.486071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518944079694082:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:29.486149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:29.326622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518945677957719:2072] [node 4] ICS09 handshake done sender: [6:7571518945677957711:2120] self: [6:7571518945677957715:0] peer: [4:7571518939784726764:0] socket: 502 qp: -1
2025-11-11T17:20:29.172499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518941368218032:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.518065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:29.518136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518942170623025:2062] [node 4] ICP22 created new session: [3:7571518946465590358:2062]
2025-11-11T17:20:29.518222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518946465590358:2062] [node 4] ICS09 handshake done sender: [3:7571518946465590335:2117] self: [3:7571518946465590334:0] peer: [4:7571518944079694077:0] socket: 451 qp: -1
2025-11-11T17:20:29.518270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518946465590358:2062] [node 4] ICS10 traffic start
2025-11-11T17:20:29.518321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:29.518828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518942170623025:2062] [node 4] ICP35 event processing took too much time PendingConnection:278(44253885ns) {HandleHandshakeStatus:329(36797901ns) {DropIncomingHandshake:441(29306ns) DropOutgoingHandshake:456(767ns) SwitchToState:178(1600ns) ProcessPendingSessionEvents:495(112838ns) {ForwardSessionEventToSession:612(62774ns) ForwardSessionEventToSession:612(3654ns) ForwardSessionEventToSession:612(3360ns) ForwardSessionEventToSession:612(6041ns) ForwardSessionEventToSession:612(5883ns) ForwardSessionEventToSession:612(2886ns) ForwardSessionEventToSession:612(6453ns) ForwardSessionEventToSession:612(3113ns) ForwardSessionEventToSession:612(5832ns) ForwardSessionEventToSession:612(2431ns)}}}
2025-11-11T17:20:29.326665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518945677957719:2072] [node 4] ICS10 traffic start
2025-11-11T17:20:29.326721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:29.327269Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518941382990405:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:29.327623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518941382990327:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.327696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:29.327745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518941382990327:2049] [node 2] ICP22 created new session: [6:7571518945677957721:2049]
2025-11-11T17:20:29.327789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518945677957721:2049] [node 2] ICS09 handshake done sender: [6:7571518941382990405:2116] self: [6:7571518945677957705:0] peer: [2:7571518941219799059:0] socket: 477 qp: -1
2025-11-11T17:20:29.327833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518945677957721:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:29.327864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:29.338018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518945677957718:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:29.409868Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518945677957718:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.433685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518941382990328:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.447608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518941382990328:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(23708452ns) {IncomingHandshake:261(19883344ns) {DropIncomingHandshake:441(1125ns) PrepareNewSessionHandshake:476(1740ns) {SwitchToState:178(887ns)} GenerateSessionVirtualId:988(649ns) IssueIncomingHandshakeReply:193(7875095ns) {SwitchToState:178(2025ns)}}}
2025-11-11T17:20:29.459233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518945677957724:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33347
2025-11-11T17:20:29.498701Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518945677957718:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:29.499082Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518944675365267:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:29.530477Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12746) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.531657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518939784726686:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12746) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518944079694070:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:29.538112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518944079694084:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58401
2025-11-11T17:20:29.509737Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.534283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518941382990328:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518945677957718:2123] held: no
2025-11-11T17:20:29.540984Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518944079694070:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:29.541243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518939784726686:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.541514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:29.541576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518939784726686:2048] [node 1] ICP22 created new session: [4:7571518944079694085:2048]
2025-11-11T17:20:29.541649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518944079694085:2048] [node 1] ICS09 handshake done sender: [4:7571518944079694070:2119] self: [4:7571518944079694080:0] peer: [1:7571518944675365266:0] socket: 508 qp: -1
2025-11-11T17:20:29.541701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518944079694085:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:29.541803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:29.548440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518941219799049:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(154397194ns) {HandleHandshakeStatus:329(154392542ns) {DropIncomingHandshake:441(31663ns) DropOutgoingHandshake:456(584ns) SwitchToState:178(511ns) ProcessPendingSessionEvents:495(45683054ns) {ForwardSessionEventToSession:612(31584778ns) ForwardSessionEventToSession:612(7853385ns) ForwardSessionEventToSession:612(23331ns) ForwardSessionEventToSession:612(8719ns) ForwardSessionEventToSession:612(8012ns) ForwardSessionEventToSession:612(3075ns) ForwardSessionEventToSession:612(2765ns) ForwardSessionEventToSession:612(5456ns) ForwardSessionEventToSession:612(2821ns) ForwardSessionEventToSession:612(5717ns)}}}
2025-11-11T17:20:29.218101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518945663185421:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:29.527031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518940380397950:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:29.549015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:29.549080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518940380397950:2103] [node 4] ICP22 created new session: [1:7571518944675365276:2103]
2025-11-11T17:20:29.549131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518944675365276:2103] [node 4] ICS09 handshake done sender: [1:7571518944675365267:2119] self: [1:7571518944675365266:0] peer: [4:7571518944079694080:0] socket: 507 qp: -1
2025-11-11T17:20:29.549166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518944675365276:2103] [node 4] ICS10 traffic start
2025-11-11T17:20:29.549245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:29.554048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518940380397950:2103] [node 4] ICP35 event processing took too much time PendingConnection:278(39643754ns) {HandleHandshakeStatus:329(39641726ns) {DropIncomingHandshake:441(32842ns) DropOutgoingHandshake:456(563ns) SwitchToState:178(1045ns) ProcessPendingSessionEvents:495(4376360ns) {ForwardSessionEventToSession:612(4314227ns) ForwardSessionEventToSession:612(6544ns) ForwardSessionEventToSession:612(6447ns) ForwardSessionEventToSession:612(3043ns) ForwardSessionEventToSession:612(9899ns) ForwardSessionEventToSession:612(2966ns) ForwardSessionEventToSession:612(5474ns) ForwardSessionEventToSession:612(7886ns) ForwardSessionEventToSession:612(3206ns) ForwardSessionEventToSession:612(2452ns)}}}
2025-11-11T17:20:29.237951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518945663185422:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:29.259220Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518945663185422:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:29.265703Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518941368218118:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:29.267065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518941368218031:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:29.281693Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.281927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518941368218032:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518941368218118:2119] held: no
2025-11-11T17:20:29.305710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518941368218032:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.309726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:29.325677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518941368218032:2050] [node 3] ICP22 created new session: [5:7571518945663185424:2050]
2025-11-11T17:20:29.366139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518945663185424:2050] [node 3] ICS09 handshake done sender: [5:7571518941368218118:2119] self: [5:7571518945663185420:0] peer: [3:7571518942170623029:0] socket: 474 qp: -1
2025-11-11T17:20:29.366215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518945663185424:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:29.366312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:29.366596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518941368218032:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(60795370ns) {HandleHandshakeStatus:329(60791807ns) {DropIncomingHandshake:441(634ns) DropOutgoingHandshake:456(425ns) SwitchToState:178(7117ns) ProcessPendingSessionEvents:495(1310ns)}}
2025-11-11T17:20:29.421793Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13883) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:29.423366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518941368218031:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13883) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518945663185422:2122] held: no
2025-11-11T17:20:29.485774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518945663185425:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25842
2025-11-11T17:20:29.498304Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518945663185422:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:29.506022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518941368218031:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.527124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:29.534202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518941368218031:2049] [node 2] ICP22 created new session: [5:7571518945663185427:2049]
2025-11-11T17:20:29.546975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518945663185427:2049] [node 2] ICS09 handshake done sender: [5:7571518945663185422:2122] self: [5:7571518945663185423:0] peer: [2:7571518945514766369:0] socket: 494 qp: -1
2025-11-11T17:20:29.547034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518945663185427:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:29.547133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:29.547469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518941368218031:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(41340132ns) {HandleHandshakeStatus:329(41337148ns) {DropIncomingHandshake:441(571ns) DropOutgoingHandshake:456(383ns) SwitchToState:178(572ns) ProcessPendingSessionEvents:495(965ns)}}
2025-11-11T17:20:29.556512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518941382990328:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(42592973ns) {HandleHandshakeStatus:395(42589696ns) {LogHandshakeFail:472(19931ns) {UpdateErrorStateLog:544(5662ns)} DropOutgoingHandshake:456(2400ns) ProcessPendingSessionEvents:495(21920811ns) {DropSessionEvent:512(29811ns) DropSessionEvent:512(11412ns) DropSessionEvent:512(11722ns) DropSessionEvent:512(12686ns) DropSessionEvent:512(6461ns) DropSessionEvent:512(6227ns) DropSessionEvent:512(6409ns) DropSessionEvent:512(3651016ns) DropSessionEvent:512(27625ns) DropSessionEvent:512(11254ns)}}}
2025-11-11T17:20:29.556662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518941382990328:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:29.556754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:29.556809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518941382990328:2050] [node 3] ICP22 created new session: [6:7571518945677957725:2050]
2025-11-11T17:20:29.556877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518945677957725:2050] [node 3] ICS09 handshake done sender: [6:7571518945677957718:2123] self: [6:7571518945677957723:0] peer: [3:7571518946465590340:0] socket: 491 qp: -1
2025-11-11T17:20:29.556916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518945677957725:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:29.556958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:29.627441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:29.627740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518941219799067:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-11-11T17:20:29.627785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518941219798986:2048] [node 1] ICP30 unregister session Session# [2:7571518941219799067:2048] VirtualId# [2:7571518941219799062:0]
2025-11-11T17:20:29.627824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518941219799067:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.627932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:29.627993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:29.645732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:29.633738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:29.651049Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:29.646341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:29.646017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518945677957713:2048] [node 1] ICS01 socket: 473 reason# EndOfStream
2025-11-11T17:20:29.641709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518946465590352:2048] [node 1] ICS01 socket: 499 reason# EndOfStream
2025-11-11T17:20:29.646063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518941382990326:2048] [node 1] ICP30 unregister session Session# [6:7571518945677957713:2048] VirtualId# [6:7571518945677957706:0]
2025-11-11T17:20:29.646101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518945677957713:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.650705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:29.650884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:29.681992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518944079694085:2048] [node 1] ICS01 socket: 508 reason# EndOfStream
2025-11-11T17:20:29.682476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518939784726686:2048] [node 1] ICP30 unregister session Session# [4:7571518944079694085:2048] VirtualId# [4:7571518944079694080:0]
2025-11-11T17:20:29.682506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518944079694085:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.697660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:29.647187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518942170622958:2048] [node 1] ICP30 unregister session Session# [3:7571518946465590352:2048] VirtualId# [3:7571518946465590339:0]
2025-11-11T17:20:29.650911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518946465590352:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.650984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:29.651020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:29.682072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518945663185416:2048] [node 1] ICS01 socket: 459 reason# EndOfStream
2025-11-11T17:20:29.682130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518941368218030:2048] [node 1] ICP30 unregister session Session# [5:7571518945663185416:2048] VirtualId# [5:7571518941368218114:0]
2025-11-11T17:20:29.682173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518945663185416:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.682302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:29.682351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:29.702017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:29.762458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:29.762713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518946465590348:2049] [node 2] ICS01 socket: 496 reason# EndOfStream
2025-11-11T17:20:29.762753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518942170622959:2049] [node 2] ICP30 unregister session Session# [3:7571518946465590348:2049] VirtualId# [3:7571518946465590337:0]
2025-11-11T17:20:29.762788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518946465590348:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.762878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:29.762917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:29.773826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:29.774053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518945663185427:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-11T17:20:29.774093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518941368218031:2049] [node 2] ICP30 unregister session Session# [5:7571518945663185427:2049] VirtualId# [5:7571518945663185423:0]
2025-11-11T17:20:29.774123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518945663185427:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.774213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:29.774255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:29.774537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:29.774738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518945677957721:2049] [node 2] ICS01 socket: 477 reason# EndOfStream
2025-11-11T17:20:29.774774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518941382990327:2049] [node 2] ICP30 unregister session Session# [6:7571518945677957721:2049] VirtualId# [6:7571518945677957705:0]
2025-11-11T17:20:29.774798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518945677957721:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.774860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:29.774897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:29.773787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:29.774028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518944079694078:2049] [node 2] ICS01 socket: 465 reason# EndOfStream
2025-11-11T17:20:29.774072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518939784726687:2049] [node 2] ICP30 unregister session Session# [4:7571518944079694078:2049] VirtualId# [4:7571518944079694067:0]
2025-11-11T17:20:29.774107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518944079694078:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.774185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:29.774229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:29.964692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:29.964130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:29.964376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518945663185424:2050] [node 3] ICS01 socket: 474 reason# EndOfStream
2025-11-11T17:20:29.964431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518941368218032:2050] [node 3] ICP30 unregister session Session# [5:7571518945663185424:2050] VirtualId# [5:7571518945663185420:0]
2025-11-11T17:20:29.964464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518945663185424:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.964962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:29.964550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:29.964593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:29.965200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518944079694082:2050] [node 3] ICS01 socket: 471 reason# EndOfStream
2025-11-11T17:20:29.965253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518939784726688:2050] [node 3] ICP30 unregister session Session# [4:7571518944079694082:2050] VirtualId# [4:7571518944079694077:0]
2025-11-11T17:20:29.964875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518945677957725:2050] [node 3] ICS01 socket: 491 reason# EndOfStream
2025-11-11T17:20:29.965285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518944079694082:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.965378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:29.964912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518941382990328:2050] [node 3] ICP30 unregister session Session# [6:7571518945677957725:2050] VirtualId# [6:7571518945677957723:0]
2025-11-11T17:20:29.964935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518945677957725:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:29.964987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:29.965036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:29.965435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:30.033793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:30.034026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518945677957719:2072] [node 4] ICS01 socket: 502 reason# EndOfStream
2025-11-11T17:20:30.034067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518941382990352:2072] [node 4] ICP30 unregister session Session# [6:7571518945677957719:2072] VirtualId# [6:7571518945677957715:0]
2025-11-11T17:20:30.034104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518945677957719:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:30.034196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:30.034244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:30.034346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:30.034514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518945663185418:2064] [node 4] ICS01 socket: 479 reason# EndOfStream
2025-11-11T17:20:30.034549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518941368218050:2064] [node 4] ICP30 unregister session Session# [5:7571518945663185418:2064] VirtualId# [5:7571518941368218116:0]
2025-11-11T17:20:30.034578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518945663185418:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:30.034632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:30.034668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:30.133775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:30.134035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518941382990407:2061] [node 5] ICS01 socket: 462 reason# EndOfStream
2025-11-11T17:20:30.134082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518941382990389:2061] [node 5] ICP30 unregister session Session# [6:7571518941382990407:2061] VirtualId# [6:7571518941382990404:0]
2025-11-11T17:20:30.134122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518941382990407:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:30.134241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:30.134289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:30.333408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518949315395445:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:30.333446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518949315395446:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:30.373311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518951559196660:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:30.373392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518951559196661:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:30.390166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518950666820092:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:30.390228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518950666820093:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:30.441819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518948819975470:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:30.441870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518948819975471:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:30.441900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518948819975472:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:30.464636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518947954979548:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:30.464676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518947954979549:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:30.464695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518947954979550:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:30.485290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518948532354738:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:30.485347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518948532354739:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:30.485476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518948532354740:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:30.746271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518948532354797:2104] [node 5] ICP01 ready to work
2025-11-11T17:20:30.749414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518948532354800:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:30.795589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518949315395508:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:30.796006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518949315395509:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:30.796343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518949315395510:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:30.865119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518950666820157:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:30.870428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518950666820158:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:30.871479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518948819975535:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:30.872701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518951559196660:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(103380096ns) {Configure:104(98200115ns) {ProcessConfigured:135(92280ns) {StartInitialHandshake:157(86610ns) {DropHandshakes:469(3866ns) {DropIncomingHandshake:441(871ns) DropOutgoingHandshake:456(1161ns)} GenerateSessionVirtualId:988(886ns) PrepareNewSessionHandshake:476(3199ns) {SwitchToState:178(1286ns)}}}}}
2025-11-11T17:20:30.872991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518948819975536:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:30.872822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518949315395446:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(58440707ns) {Configure:104(58437351ns) {ProcessConfigured:135(446848ns) {StartInitialHandshake:157(444745ns) {DropHandshakes:469(2978ns) {DropIncomingHandshake:441(295ns) DropOutgoingHandshake:456(1097ns)} GenerateSessionVirtualId:988(1157ns) PrepareNewSessionHandshake:476(1718ns) {SwitchToState:178(661ns)}}}}}
2025-11-11T17:20:30.873918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518951559196725:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:30.874303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518951559196726:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:30.873179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518950666820159:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:30.874571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518951559196727:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:30.876372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518947954979548:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(73597114ns) {Configure:104(73589265ns) {ProcessConfigured:135(317578ns) {StartInitialHandshake:157(315019ns) {DropHandshakes:469(2963ns) {DropIncomingHandshake:441(648ns) DropOutgoingHandshake:456(568ns)} GenerateSessionVirtualId:988(668ns) PrepareNewSessionHandshake:476(1922ns) {SwitchToState:178(730ns)}}}}}
2025-11-11T17:20:30.876995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518947954979614:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.877319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518947954979615:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:30.883893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518947954979613:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:30.890195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518948532354739:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20850352ns) {Configure:104(20842536ns) {ProcessConfigured:135(1163950ns) {StartInitialHandshake:157(1162547ns) {DropHandshakes:469(1603ns) {DropIncomingHandshake:441(516ns) DropOutgoingHandshake:456(336ns)} GenerateSessionVirtualId:988(674ns) PrepareNewSessionHandshake:476(1995ns) {SwitchToState:178(1184ns)}}}}}
2025-11-11T17:20:30.894443Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518950666820092:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19184309ns) {Configure:104(19178351ns) {ProcessConfigured:135(288965ns) {StartInitialHandshake:157(287471ns) {DropHandshakes:469(1156ns) {DropIncomingHandshake:441(252ns) DropOutgoingHandshake:456(462ns)} GenerateSessionVirtualId:988(423ns) PrepareNewSessionHandshake:476(872ns) {SwitchToState:178(355ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.898852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518951559196736:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.905119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518951559196740:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-11T17:20:30.911922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196736:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.912549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518949315395521:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64555
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.921964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518951559196742:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.923496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518948819975536:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27909957ns) {Configure:104(27907192ns) {ProcessConfigured:135(5364841ns) {StartInitialHandshake:157(5360104ns) {DropHandshakes:469(3620ns) {DropIncomingHandshake:441(839ns) DropOutgoingHandshake:456(808ns)} GenerateSessionVirtualId:988(1017ns) PrepareNewSessionHandshake:476(4072ns) {SwitchToState:178(1538ns)}}}}}
2025-11-11T17:20:30.923390Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196742:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.923577Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518947954979613:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.923981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518950666820157:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25772953ns) {Configure:104(25770727ns) {ProcessConfigured:135(4613817ns) {StartInitialHandshake:157(4612551ns) {DropHandshakes:469(4303ns) {DropIncomingHandshake:441(444ns) DropOutgoingHandshake:456(324ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:476(2132ns) {SwitchToState:178(387ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.925965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518949315395524:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64555
2025-11-11T17:20:30.926291Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395521:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.926133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518948819975533:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54048
2025-11-11T17:20:30.931270Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395524:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.942798Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518948819975533:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.943741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518948819975547:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54048
2025-11-11T17:20:30.943869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518948819975548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54048
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.945271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518950666820159:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18515158ns) {Configure:104(18509969ns) {ProcessConfigured:135(68648ns) {StartInitialHandshake:157(65288ns) {DropHandshakes:469(3397ns) {DropIncomingHandshake:441(362ns) DropOutgoingHandshake:456(608ns)} GenerateSessionVirtualId:988(659ns) PrepareNewSessionHandshake:476(1013ns) {SwitchToState:178(376ns)}}}}}
2025-11-11T17:20:30.946120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518950666820166:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1565
2025-11-11T17:20:30.946291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518950666820167:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1565
2025-11-11T17:20:30.946397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518950666820168:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1565
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.963123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518948819975553:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54048
2025-11-11T17:20:30.954695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518947954979628:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:30.968724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518950666820166:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.978406Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518947954979628:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.985832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518947954979614:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:30.981693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518948819975547:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.989046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518947954979630:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:30.988068Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196731:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.994995Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518947954979628:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:30.993626Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518950666820165:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.993681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518950666820165:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:30.994283Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518950666820168:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.994604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518950666820092:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:30.995234Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518950666820165:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:30.995569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518950666820157:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:30.995638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:30.995689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518950666820157:2107] [node 4] ICP22 created new session: [3:7571518950666820179:2107]
2025-11-11T17:20:30.995736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518950666820179:2107] [node 4] ICS09 handshake done sender: [3:7571518950666820165:2114] self: [3:7571518950666820164:0] peer: [4:7571518948819975556:0] socket: 487 qp: -1
2025-11-11T17:20:30.995767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518950666820179:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:30.995805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:30.981905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518948819975471:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:30.988740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518951559196731:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.986295Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518948819975553:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.017820Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395516:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.017880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518949315395516:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.008972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518947954979614:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:30.986473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518948819975540:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:30.986495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518948819975540:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:30.987255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518948819975472:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:30.987507Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518948819975540:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:30.988352Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518948819975548:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:30.988395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518948819975535:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.014046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322112:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.009089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:31.033851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518952827322112:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.034040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518948532354797:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.035842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322114:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.036697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518952827322112:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:31.043429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518948532354797:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.043557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:31.043615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518948532354797:2104] [node 5] ICP22 created new session: [6:7571518952827322115:2104]
2025-11-11T17:20:31.043661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518952827322115:2104] [node 5] ICS09 handshake done sender: [6:7571518952827322112:2115] self: [6:7571518952827322113:0] peer: [5:7571518947954979622:0] socket: 496 qp: -1
2025-11-11T17:20:31.043700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518952827322115:2104] [node 5] ICS10 traffic start
2025-11-11T17:20:31.043743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:31.049358Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196740:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.042311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518950666820167:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.009148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518947954979614:2108] [node 4] ICP22 created new session: [5:7571518952249946927:2108]
2025-11-11T17:20:30.988477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.049640Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196729:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.049672Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518951559196729:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.050721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518951559196744:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-11T17:20:31.050910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518951559196745:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-11T17:20:31.051077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518951559196660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.051467Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518951559196731:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:31.051950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518951559196725:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.052022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:31.052070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518951559196725:2107] [node 4] ICP22 created new session: [2:7571518955854164045:2107]
2025-11-11T17:20:31.052121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518955854164045:2107] [node 4] ICS09 handshake done sender: [2:7571518951559196731:2112] self: [2:7571518951559196730:0] peer: [4:7571518948819975554:0] socket: 468 qp: -1
2025-11-11T17:20:31.052156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518955854164045:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:31.052226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:31.057816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518951559196729:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:31.009195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518952249946927:2108] [node 4] ICS09 handshake done sender: [5:7571518947954979628:2116] self: [5:7571518947954979629:0] peer: [4:7571518948819975539:0] socket: 483 qp: -1
2025-11-11T17:20:30.988586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518948819975535:2108] [node 5] ICP22 created new session: [4:7571518948819975557:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.042945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518950666820093:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.009235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518952249946927:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.054556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518949315395526:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64555
2025-11-11T17:20:31.054703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518949315395527:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64555
2025-11-11T17:20:31.054875Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395514:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.054895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518949315395514:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.055624Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395512:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.055643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518949315395512:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.056774Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518949315395516:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:31.009273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:31.034409Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518947954979623:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.034435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518947954979623:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.036052Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518947954979623:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:31.036976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518947954979615:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.037080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:31.037135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518947954979615:2109] [node 6] ICP22 created new session: [5:7571518952249946931:2109]
2025-11-11T17:20:31.037183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518952249946931:2109] [node 6] ICS09 handshake done sender: [5:7571518947954979623:2115] self: [5:7571518947954979622:0] peer: [6:7571518952827322113:0] socket: 495 qp: -1
2025-11-11T17:20:31.037220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518952249946931:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:31.037260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:31.076036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518952249946933:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:31.043040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518954961787477:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1565
2025-11-11T17:20:30.988636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518948819975557:2108] [node 5] ICS09 handshake done sender: [4:7571518948819975540:2113] self: [4:7571518948819975539:0] peer: [5:7571518947954979629:0] socket: 481 qp: -1
2025-11-11T17:20:30.989701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518948819975557:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:30.989782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:31.019028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518948819975470:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.019436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518953114942855:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54048
2025-11-11T17:20:31.076480Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518948819975553:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.062969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518954961787480:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1565
2025-11-11T17:20:31.074595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518954961787481:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1565
2025-11-11T17:20:31.083238Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518950666820167:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:31.090284Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518950666820168:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:31.084163Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518949315395514:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:31.091462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322117:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.101871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518950666820093:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.110244Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196745:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.100847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518948819975472:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.101965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:31.102010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518950666820093:2049] [node 2] ICP22 created new session: [3:7571518954961787482:2049]
2025-11-11T17:20:31.102049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518954961787482:2049] [node 2] ICS09 handshake done sender: [3:7571518950666820167:2116] self: [3:7571518954961787479:0] peer: [2:7571518951559196728:0] socket: 476 qp: -1
2025-11-11T17:20:31.102082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518954961787482:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:31.102148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:31.100047Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518949315395512:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:31.110358Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196744:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.110397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518951559196661:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.110470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:31.110529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518951559196661:2049] [node 3] ICP22 created new session: [2:7571518955854164049:2049]
2025-11-11T17:20:31.110581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518955854164049:2049] [node 3] ICS09 handshake done sender: [2:7571518951559196729:2111] self: [2:7571518951559196728:0] peer: [3:7571518954961787479:0] socket: 466 qp: -1
2025-11-11T17:20:31.110616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518955854164049:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:31.110687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:31.112934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518951559196661:2049] [node 3] ICP18 handshake (actor [2:7571518951559196745:2119]) is too old
2025-11-11T17:20:31.113074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518955854164048:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.100929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:31.120642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395527:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.120703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518949315395446:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.125700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:31.138670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518950666820093:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19852352ns) {HandleHandshakeStatus:329(19849887ns) {DropIncomingHandshake:441(431ns) DropOutgoingHandshake:456(4988ns) SwitchToState:178(552ns) ProcessPendingSessionEvents:495(3736236ns) {ForwardSessionEventToSession:612(3681911ns) ForwardSessionEventToSession:612(6319ns) ForwardSessionEventToSession:612(3317ns) ForwardSessionEventToSession:612(7256ns) ForwardSessionEventToSession:612(2451ns) ForwardSessionEventToSession:612(6341ns) ForwardSessionEventToSession:612(5898ns) ForwardSessionEventToSession:612(3103ns) ForwardSessionEventToSession:612(4853ns) ForwardSessionEventToSession:612(3246ns)}}}
2025-11-11T17:20:31.139287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518950666820092:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.139376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:31.139412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518950666820092:2048] [node 1] ICP22 created new session: [3:7571518954961787484:2048]
2025-11-11T17:20:31.139460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518954961787484:2048] [node 1] ICS09 handshake done sender: [3:7571518950666820168:2117] self: [3:7571518950666820178:0] peer: [1:7571518949315395515:0] socket: 477 qp: -1
2025-11-11T17:20:31.139498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518954961787484:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:31.139571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.142131Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:31.144999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518950666820157:2107] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.146076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518952249946934:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:31.138451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518949315395446:2049] [node 3] ICP22 created new session: [1:7571518953610362829:2049]
2025-11-11T17:20:31.141276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518953610362829:2049] [node 3] ICS09 handshake done sender: [1:7571518949315395516:2112] self: [1:7571518949315395515:0] peer: [3:7571518950666820178:0] socket: 474 qp: -1
2025-11-11T17:20:31.141318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518953610362829:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:31.141384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:31.150351Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518952249946934:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.150507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518947954979550:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.100981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518948819975472:2050] [node 3] ICP22 created new session: [4:7571518953114942860:2050]
2025-11-11T17:20:31.101021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518953114942860:2050] [node 3] ICS09 handshake done sender: [4:7571518948819975553:2119] self: [4:7571518948819975556:0] peer: [3:7571518950666820164:0] socket: 488 qp: -1
2025-11-11T17:20:31.101055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518953114942860:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:31.101127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:31.125447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518953114942858:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54048
2025-11-11T17:20:31.141047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518953114942859:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54048
2025-11-11T17:20:31.150508Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518952827322117:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.150646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518948532354740:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.165193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322120:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.150893Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518950666820172:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.149847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518951559196725:2107] [node 4] ICP18 handshake (actor [2:7571518951559196744:2118]) is too old
2025-11-11T17:20:31.176921Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:31.190193Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518948819975547:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:31.190566Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518948819975548:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:31.190833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518948819975471:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518948819975547:2117] held: no
2025-11-11T17:20:31.180301Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518952249946933:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.197836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518947954979549:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.150934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518950666820172:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.210184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322121:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.173986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518949315395446:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(20890986ns) {HandleHandshakeStatus:329(20887389ns) {DropIncomingHandshake:441(683ns) DropOutgoingHandshake:456(943ns) SwitchToState:178(784ns) ProcessPendingSessionEvents:495(86278ns) {ForwardSessionEventToSession:612(41560ns) ForwardSessionEventToSession:612(2544ns) ForwardSessionEventToSession:612(2257ns) ForwardSessionEventToSession:612(4995ns) ForwardSessionEventToSession:612(2659ns) ForwardSessionEventToSession:612(2669ns) ForwardSessionEventToSession:612(5703ns) ForwardSessionEventToSession:612(5771ns) ForwardSessionEventToSession:612(6085ns) ForwardSessionEventToSession:612(2574ns)}}}
2025-11-11T17:20:31.174243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518949315395508:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.174309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:31.174363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518949315395508:2107] [node 4] ICP22 created new session: [1:7571518953610362832:2107]
2025-11-11T17:20:31.174407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518953610362832:2107] [node 4] ICS09 handshake done sender: [1:7571518949315395514:2111] self: [1:7571518949315395513:0] peer: [4:7571518953114942857:0] socket: 470 qp: -1
2025-11-11T17:20:31.174446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518953610362832:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:31.174484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:31.175063Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395526:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.175101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518949315395445:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.175186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:31.175229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518949315395445:2048] [node 2] ICP22 created new session: [1:7571518953610362834:2048]
2025-11-11T17:20:31.175295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518953610362834:2048] [node 2] ICS09 handshake done sender: [1:7571518949315395512:2110] self: [1:7571518949315395511:0] peer: [2:7571518955854164044:0] socket: 467 qp: -1
2025-11-11T17:20:31.175331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518953610362834:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:31.175385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:31.202700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518949315395508:2107] [node 4] ICP18 handshake (actor [1:7571518949315395526:2117]) is too old
2025-11-11T17:20:31.203217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518949315395446:2049] [node 3] ICP18 handshake (actor [1:7571518949315395527:2118]) is too old
2025-11-11T17:20:31.190948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518948819975471:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.165681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518951559196740:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.191019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:31.217869Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518950666820170:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.217923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518947954979549:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19834490ns) {IncomingHandshake:261(19832307ns) {DropIncomingHandshake:441(530ns) PrepareNewSessionHandshake:476(2232ns) {SwitchToState:178(1047ns)} GenerateSessionVirtualId:988(700ns) IssueIncomingHandshakeReply:193(7866240ns) {SwitchToState:178(10982ns)}}}
2025-11-11T17:20:31.230660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518952249946937:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:31.246117Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:31.191068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518948819975471:2049] [node 2] ICP22 created new session: [4:7571518953114942862:2049]
2025-11-11T17:20:31.233743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518953610362831:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64555
2025-11-11T17:20:31.217919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518950666820170:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.264299Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518953610362831:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.261257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322122:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.261345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322123:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.191115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518953114942862:2049] [node 2] ICS09 handshake done sender: [4:7571518948819975547:2117] self: [4:7571518948819975554:0] peer: [2:7571518951559196730:0] socket: 478 qp: -1
2025-11-11T17:20:31.261921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518952249946938:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:31.262104Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518952249946934:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:31.266611Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518952827322123:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.266724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518952827322121:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.266823Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518952827322117:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:31.229958Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518950666820172:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:31.267325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518948532354740:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.265489Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518950666820170:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:31.266049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518950666820159:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.266144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:31.266195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518950666820159:2109] [node 5] ICP22 created new session: [3:7571518954961787489:2109]
2025-11-11T17:20:31.266243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518954961787489:2109] [node 5] ICS09 handshake done sender: [3:7571518950666820172:2119] self: [3:7571518950666820171:0] peer: [5:7571518952249946935:0] socket: 485 qp: -1
2025-11-11T17:20:31.266290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518954961787489:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:31.268588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:31.274235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518950666820158:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.274324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:31.274398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518950666820158:2108] [node 6] ICP22 created new session: [3:7571518954961787491:2108]
2025-11-11T17:20:31.274452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518954961787491:2108] [node 6] ICS09 handshake done sender: [3:7571518950666820170:2118] self: [3:7571518950666820169:0] peer: [6:7571518952827322119:0] socket: 502 qp: -1
2025-11-11T17:20:31.274488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518954961787491:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:31.274527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:31.191148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518953114942862:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:31.191194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:31.191665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518948819975470:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.191732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:31.191770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518948819975470:2048] [node 1] ICP22 created new session: [4:7571518953114942865:2048]
2025-11-11T17:20:31.191846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518953114942865:2048] [node 1] ICS09 handshake done sender: [4:7571518948819975548:2118] self: [4:7571518953114942857:0] peer: [1:7571518949315395513:0] socket: 479 qp: -1
2025-11-11T17:20:31.191868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518953114942865:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:31.191917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:31.277442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518951559196660:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.275348Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518952249946938:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.277465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:31.277560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518948532354740:2050] [node 3] ICP22 created new session: [6:7571518952827322124:2050]
2025-11-11T17:20:31.277634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518952827322124:2050] [node 3] ICS09 handshake done sender: [6:7571518952827322117:2117] self: [6:7571518952827322119:0] peer: [3:7571518950666820169:0] socket: 456 qp: -1
2025-11-11T17:20:31.277680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518952827322124:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:31.277780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:31.281008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518948532354738:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.281105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518948532354800:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.277546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:31.277615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518951559196660:2048] [node 1] ICP22 created new session: [2:7571518955854164052:2048]
2025-11-11T17:20:31.277667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518955854164052:2048] [node 1] ICS09 handshake done sender: [2:7571518951559196740:2116] self: [2:7571518955854164044:0] peer: [1:7571518949315395511:0] socket: 469 qp: -1
2025-11-11T17:20:31.275419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518947954979550:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.277702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518955854164052:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:31.277754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:31.275498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:31.283549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571518951559196660:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:31.284112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196733:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.284139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518951559196733:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.285064Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395520:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.285099Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518949315395520:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.275559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518947954979550:2050] [node 3] ICP22 created new session: [5:7571518952249946939:2050]
2025-11-11T17:20:31.275607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518952249946939:2050] [node 3] ICS09 handshake done sender: [5:7571518952249946934:2119] self: [5:7571518952249946935:0] peer: [3:7571518950666820171:0] socket: 486 qp: -1
2025-11-11T17:20:31.275646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518952249946939:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:31.275676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.284829Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518948819975546:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.300948Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518951559196733:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:31.284871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518948819975546:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.301440Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518948819975546:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:31.294506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518947954979548:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.294928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518952249946941:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:31.301721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518948819975536:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.285967Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518949315395520:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:31.306146Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8403) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:31.301770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:31.301809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518948819975536:2109] [node 6] ICP22 created new session: [4:7571518953114942868:2109]
2025-11-11T17:20:31.301863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518953114942868:2109] [node 6] ICS09 handshake done sender: [4:7571518948819975546:2116] self: [4:7571518948819975545:0] peer: [6:7571518952827322127:0] socket: 471 qp: -1
2025-11-11T17:20:31.301890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518953114942868:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:31.301946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:31.310164Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518949315395518:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.310214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518949315395518:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.311175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518949315395510:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.311271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:31.311327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518949315395510:2109] [node 6] ICP22 created new session: [1:7571518953610362840:2109]
2025-11-11T17:20:31.311374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518953610362840:2109] [node 6] ICS09 handshake done sender: [1:7571518949315395520:2114] self: [1:7571518949315395519:0] peer: [6:7571518952827322126:0] socket: 516 qp: -1
2025-11-11T17:20:31.311414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518953610362840:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:31.311482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:31.323263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518951559196726:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.323419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:31.323471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518951559196726:2108] [node 5] ICP22 created new session: [2:7571518955854164055:2108]
2025-11-11T17:20:31.323556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518955854164055:2108] [node 5] ICS09 handshake done sender: [2:7571518951559196733:2113] self: [2:7571518951559196732:0] peer: [5:7571518952249946936:0] socket: 505 qp: -1
2025-11-11T17:20:31.323593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518955854164055:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:31.323633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:31.324715Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518952249946933:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:31.325244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518948532354800:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8403) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518952827322121:2119] held: no
2025-11-11T17:20:31.325435Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:31.326490Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518949315395518:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:31.329377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518949315395509:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.326973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518947954979549:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.329482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:31.327069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:31.329523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518949315395509:2108] [node 5] ICP22 created new session: [1:7571518953610362843:2108]
2025-11-11T17:20:31.329572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518953610362843:2108] [node 5] ICS09 handshake done sender: [1:7571518949315395518:2113] self: [1:7571518949315395517:0] peer: [5:7571518952249946942:0] socket: 515 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.330246Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518952827322120:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.330374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322128:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.329624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518953610362843:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:31.329655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:31.327119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518947954979549:2049] [node 2] ICP22 created new session: [5:7571518952249946944:2049]
2025-11-11T17:20:31.327164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518952249946944:2049] [node 2] ICS09 handshake done sender: [5:7571518952249946933:2118] self: [5:7571518952249946936:0] peer: [2:7571518951559196732:0] socket: 512 qp: -1
2025-11-11T17:20:31.327205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518952249946944:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:31.327238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.331386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518948532354738:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518952827322123:2121] held: no
2025-11-11T17:20:31.331583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322129:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.332154Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518952827322123:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.332515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518952249946943:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57
2025-11-11T17:20:31.334338Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518952249946938:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:31.334775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518947954979548:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.334863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:31.334916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518947954979548:2048] [node 1] ICP22 created new session: [5:7571518952249946946:2048]
2025-11-11T17:20:31.334956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518952249946946:2048] [node 1] ICS09 handshake done sender: [5:7571518952249946938:2121] self: [5:7571518952249946942:0] peer: [1:7571518949315395517:0] socket: 518 qp: -1
2025-11-11T17:20:31.334985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518952249946946:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:31.335031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:31.338675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518948532354738:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.338762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:31.338808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518948532354738:2048] [node 1] ICP22 created new session: [6:7571518952827322130:2048]
2025-11-11T17:20:31.338849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518952827322130:2048] [node 1] ICS09 handshake done sender: [6:7571518952827322123:2121] self: [6:7571518952827322126:0] peer: [1:7571518949315395519:0] socket: 519 qp: -1
2025-11-11T17:20:31.338888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518952827322130:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:31.338925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:31.339504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518948532354739:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:31.339829Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518952827322121:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:31.342139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518948532354800:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.342257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:31.342318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518948532354800:2107] [node 4] ICP22 created new session: [6:7571518952827322133:2107]
2025-11-11T17:20:31.342375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518952827322133:2107] [node 4] ICS09 handshake done sender: [6:7571518952827322121:2119] self: [6:7571518952827322127:0] peer: [4:7571518948819975545:0] socket: 480 qp: -1
2025-11-11T17:20:31.342414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518952827322133:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:31.342452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:31.343214Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518951559196735:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:31.343250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518951559196735:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:31.344655Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518951559196735:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:31.345020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518952827322135:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25871
2025-11-11T17:20:31.345330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518951559196727:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:31.345584Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518952827322120:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:31.345731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:31.345787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518951559196727:2109] [node 6] ICP22 created new session: [2:7571518955854164058:2109]
2025-11-11T17:20:31.345830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518955854164058:2109] [node 6] ICS09 handshake done sender: [2:7571518951559196735:2114] self: [2:7571518951559196734:0] peer: [6:7571518952827322132:0] socket: 489 qp: -1
2025-11-11T17:20:31.345863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518955854164058:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:31.345928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:31.346920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518948532354739:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:31.347024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:31.347077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518948532354739:2049] [node 2] ICP22 created new session: [6:7571518952827322136:2049]
2025-11-11T17:20:31.347118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518952827322136:2049] [node 2] ICS09 handshake done sender: [6:7571518952827322120:2118] self: [6:7571518952827322132:0] peer: [2:7571518951559196734:0] socket: 498 qp: -1
2025-11-11T17:20:31.347149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518952827322136:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:31.347205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:31.369417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:31.369685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571518955854164052:2048] [node 1] ICS07 socket disconnect 469 reason# EndOfStream
2025-11-11T17:20:31.369718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518955854164052:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.369799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:31.369843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518955854164052:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:31.373768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:31.374038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518954961787484:2048] [node 1] ICS01 socket: 477 reason# EndOfStream
2025-11-11T17:20:31.374084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518950666820092:2048] [node 1] ICP30 unregister session Session# [3:7571518954961787484:2048] VirtualId# [3:7571518950666820178:0]
2025-11-11T17:20:31.374124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518954961787484:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.374242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:31.374293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:31.378900Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:20:31.380045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518951559196660:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:20:31.380085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571518955854164052:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:31.381047Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Couldn't connect to any resolved address
2025-11-11T17:20:31.381298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518951559196660:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:31.381348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518955854164052:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:31.381382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518951559196660:2048] [node 1] ICP30 unregister session Session# [2:7571518955854164052:2048] VirtualId# [2:7571518955854164044:0]
2025-11-11T17:20:31.381441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:31.381473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571518951559196660:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:31.381488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11260) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:31.389785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:31.389998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518952827322130:2048] [node 1] ICS01 socket: 519 reason# EndOfStream
2025-11-11T17:20:31.390042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518948532354738:2048] [node 1] ICP30 unregister session Session# [6:7571518952827322130:2048] VirtualId# [6:7571518952827322126:0]
2025-11-11T17:20:31.390077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518952827322130:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.390141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:31.390181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:31.389885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:31.390046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518952249946946:2048] [node 1] ICS01 socket: 518 reason# EndOfStream
2025-11-11T17:20:31.390075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518947954979548:2048] [node 1] ICP30 unregister session Session# [5:7571518952249946946:2048] VirtualId# [5:7571518952249946942:0]
2025-11-11T17:20:31.390097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518952249946946:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.390160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:31.390197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:31.390403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:31.390861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518953114942865:2048] [node 1] ICS01 socket: 479 reason# EndOfStream
2025-11-11T17:20:31.390890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518948819975470:2048] [node 1] ICP30 unregister session Session# [4:7571518953114942865:2048] VirtualId# [4:7571518953114942857:0]
2025-11-11T17:20:31.390920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518953114942865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.390986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:31.391024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:31.397732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571518951559196660:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:31.451288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:31.451576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518954961787482:2049] [node 2] ICS01 socket: 476 reason# EndOfStream
2025-11-11T17:20:31.451622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518950666820093:2049] [node 2] ICP30 unregister session Session# [3:7571518954961787482:2049] VirtualId# [3:7571518954961787479:0]
2025-11-11T17:20:31.451663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518954961787482:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.451797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:31.451844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:31.465776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:31.465985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518952249946944:2049] [node 2] ICS01 socket: 512 reason# EndOfStream
2025-11-11T17:20:31.466033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518947954979549:2049] [node 2] ICP30 unregister session Session# [5:7571518952249946944:2049] VirtualId# [5:7571518952249946936:0]
2025-11-11T17:20:31.466070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518952249946944:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.466146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:31.466186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:31.469724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:31.469960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518953114942862:2049] [node 2] ICS01 socket: 478 reason# EndOfStream
2025-11-11T17:20:31.469998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518948819975471:2049] [node 2] ICP30 unregister session Session# [4:7571518953114942862:2049] VirtualId# [4:7571518948819975554:0]
2025-11-11T17:20:31.470031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518953114942862:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.470136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:31.470181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:31.477834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:31.478134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518952827322136:2049] [node 2] ICS01 socket: 498 reason# EndOfStream
2025-11-11T17:20:31.478200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518948532354739:2049] [node 2] ICP30 unregister session Session# [6:7571518952827322136:2049] VirtualId# [6:7571518952827322132:0]
2025-11-11T17:20:31.478249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518952827322136:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.478386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:31.478463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:31.518772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:31.518935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518952249946939:2050] [node 3] ICS01 socket: 486 reason# EndOfStream
2025-11-11T17:20:31.518971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518947954979550:2050] [node 3] ICP30 unregister session Session# [5:7571518952249946939:2050] VirtualId# [5:7571518952249946935:0]
2025-11-11T17:20:31.519002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518952249946939:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.519065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:31.519107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:31.517820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:31.521767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:31.534020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518953114942860:2050] [node 3] ICS01 socket: 488 reason# EndOfStream
2025-11-11T17:20:31.534074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518948819975472:2050] [node 3] ICP30 unregister session Session# [4:7571518953114942860:2050] VirtualId# [4:7571518948819975556:0]
2025-11-11T17:20:31.534112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518953114942860:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.534243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:31.534303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:31.518029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518952827322124:2050] [node 3] ICS07 socket disconnect 456 reason# EndOfStream
2025-11-11T17:20:31.518061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518952827322124:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.518167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:31.518226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518952827322124:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:31.537723Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7430) Couldn't connect to any resolved address
2025-11-11T17:20:31.538119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518948532354740:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:31.538178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518952827322124:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:31.538213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518948532354740:2050] [node 3] ICP30 unregister session Session# [6:7571518952827322124:2050] VirtualId# [6:7571518952827322119:0]
2025-11-11T17:20:31.538268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:31.538297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518948532354740:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:31.538312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:31.549734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518948532354740:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:20:31.597787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:31.598039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518952249946927:2108] [node 4] ICS01 socket: 483 reason# EndOfStream
2025-11-11T17:20:31.598076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518947954979614:2108] [node 4] ICP30 unregister session Session# [5:7571518952249946927:2108] VirtualId# [5:7571518947954979629:0]
2025-11-11T17:20:31.598108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518952249946927:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.598221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:31.598268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:31.598370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:31.598573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518952827322133:2107] [node 4] ICS01 socket: 480 reason# EndOfStream
2025-11-11T17:20:31.598602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518948532354800:2107] [node 4] ICP30 unregister session Session# [6:7571518952827322133:2107] VirtualId# [6:7571518952827322127:0]
2025-11-11T17:20:31.598629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518952827322133:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.598682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:31.598715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:31.633794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:31.634070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518952827322115:2104] [node 5] ICS01 socket: 496 reason# EndOfStream
2025-11-11T17:20:31.634109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518948532354797:2104] [node 5] ICP30 unregister session Session# [6:7571518952827322115:2104] VirtualId# [6:7571518952827322113:0]
2025-11-11T17:20:31.634147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518952827322115:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:31.634265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:31.634315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:31.733755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518955425274535:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:31.733813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518955425274536:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:31.756271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518954044935984:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:31.756304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518954044935985:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:31.783870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518952846021300:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:31.783908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518952846021301:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:31.821922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518954975199200:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:31.821973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518954975199201:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:31.822002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518954975199202:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:31.845607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518952430622365:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:31.845647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518952430622366:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:31.845677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518952430622367:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:31.868829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518952810288415:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:31.868869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518952810288416:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:31.868881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518952810288414:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:32.405793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518958339903346:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:32.406900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518952846021300:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(92905616ns) {Configure:104(72893667ns) {ProcessConfigured:135(84374ns) {StartInitialHandshake:157(76629ns) {DropHandshakes:469(3925ns) {DropIncomingHandshake:441(802ns) DropOutgoingHandshake:456(749ns)} GenerateSessionVirtualId:988(947ns) PrepareNewSessionHandshake:476(2602ns) {SwitchToState:178(812ns)}}}}}
2025-11-11T17:20:32.422827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518958339903347:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:32.404180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518954975199200:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34161342ns) {Configure:104(34153889ns) {ProcessConfigured:135(71810ns) {StartInitialHandshake:157(69064ns) {DropHandshakes:469(2756ns) {DropIncomingHandshake:441(821ns) DropOutgoingHandshake:456(450ns)} GenerateSessionVirtualId:988(867ns) PrepareNewSessionHandshake:476(1409ns) {SwitchToState:178(435ns)}}}}}
2025-11-11T17:20:32.427314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518952810288415:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(93358901ns) {Configure:104(93350096ns) {ProcessConfigured:135(3636310ns) {StartInitialHandshake:157(3633337ns) {DropHandshakes:469(2179ns) {DropIncomingHandshake:441(527ns) DropOutgoingHandshake:456(445ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:476(10389ns) {SwitchToState:178(3365ns)}}}}}
2025-11-11T17:20:32.428285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518958339903348:2110] [node 6] ICP01 ready to work
2025-11-11T17:20:32.433890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518957140988679:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:32.449847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518956725589724:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:32.418336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518959270166562:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:32.464726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518958339903346:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(36088772ns) {Configure:104(36083552ns) {ProcessConfigured:135(98070ns) {StartInitialHandshake:157(95196ns) {DropHandshakes:469(2701ns) {DropIncomingHandshake:441(472ns) DropOutgoingHandshake:456(515ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:476(1867ns) {SwitchToState:178(687ns)}}}}}
2025-11-11T17:20:32.448184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518959720241895:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:32.426780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518959270166563:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:32.455792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518957140988681:2121] [node 6] ICP01 ready to work
2025-11-11T17:20:32.461676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518959720241894:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:32.458640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518957140988680:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:32.486312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518954975199202:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(63741975ns) {Configure:104(63735924ns) {ProcessConfigured:135(77552ns) {StartInitialHandshake:157(74126ns) {DropHandshakes:469(2921ns) {DropIncomingHandshake:441(603ns) DropOutgoingHandshake:456(454ns)} GenerateSessionVirtualId:988(961ns) PrepareNewSessionHandshake:476(1756ns) {SwitchToState:178(852ns)}}}}}
2025-11-11T17:20:32.554592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518959720241896:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:32.570316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518952430622367:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(179263521ns) {Configure:104(179258049ns) {ProcessConfigured:135(76276ns) {StartInitialHandshake:157(72531ns) {DropHandshakes:469(2001ns) {DropIncomingHandshake:441(435ns) DropOutgoingHandshake:456(365ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:476(2349ns) {SwitchToState:178(1296ns)}}}}}
2025-11-11T17:20:32.554035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518954044935985:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(127103880ns) {Configure:104(127099047ns) {ProcessConfigured:135(94094ns) {StartInitialHandshake:157(90221ns) {DropHandshakes:469(3074ns) {DropIncomingHandshake:441(846ns) DropOutgoingHandshake:456(519ns)} GenerateSessionVirtualId:988(1128ns) PrepareNewSessionHandshake:476(1871ns) {SwitchToState:178(653ns)}}}}}
2025-11-11T17:20:32.567228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518957105255777:2090] [node 5] ICP01 ready to work
2025-11-11T17:20:32.597707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518952810288416:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(276202634ns) {Configure:104(276195222ns) {ProcessConfigured:135(14022313ns) {StartInitialHandshake:157(14019375ns) {DropHandshakes:469(2995ns) {DropIncomingHandshake:441(540ns) DropOutgoingHandshake:456(961ns)} GenerateSessionVirtualId:988(822ns) PrepareNewSessionHandshake:476(5454ns) {SwitchToState:178(2461ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.598008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518954975199201:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(111441160ns) {Configure:104(111436689ns) {ProcessConfigured:135(64822ns) {StartInitialHandshake:157(61522ns) {DropHandshakes:469(8092ns) {DropIncomingHandshake:441(446ns) DropOutgoingHandshake:456(306ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:476(1204ns) {SwitchToState:178(489ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.585897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518956725589731:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:32.631146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518955425274536:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55263356ns) {Configure:104(55259985ns) {ProcessConfigured:135(111516ns) {StartInitialHandshake:157(107342ns) {DropHandshakes:469(2450ns) {DropIncomingHandshake:441(593ns) DropOutgoingHandshake:456(376ns)} GenerateSessionVirtualId:988(1009ns) PrepareNewSessionHandshake:476(2968ns) {SwitchToState:178(1079ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.591271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518952430622366:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17726472ns) {Configure:104(17723179ns) {ProcessConfigured:135(69315ns) {StartInitialHandshake:157(66230ns) {DropHandshakes:469(3153ns) {DropIncomingHandshake:441(697ns) DropOutgoingHandshake:456(537ns)} GenerateSessionVirtualId:988(1116ns) PrepareNewSessionHandshake:476(1420ns) {SwitchToState:178(613ns)}}}}}
2025-11-11T17:20:32.606158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518959270166570:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
2025-11-11T17:20:32.606374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518959270166571:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
2025-11-11T17:20:32.638826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518959270166570:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.647061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518954975199201:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.647428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518959270166575:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
2025-11-11T17:20:32.639397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518959720241897:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43329
2025-11-11T17:20:32.648143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518957105255785:2094] [node 4] ICP01 ready to work
2025-11-11T17:20:32.598822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518956725589739:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-11-11T17:20:32.639564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518959720241898:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43329
2025-11-11T17:20:32.685885Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518959270166571:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.650347Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518956725589739:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.696461Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903350:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.639862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518959720241907:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43329
2025-11-11T17:20:32.639913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518959720241908:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43329
2025-11-11T17:20:32.650094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241908:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.659808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241898:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.679770Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241907:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.685948Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241897:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.699291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518959720241896:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(109048462ns) {Configure:104(109038740ns) {ProcessConfigured:135(72734ns) {StartInitialHandshake:157(69952ns) {DropHandshakes:469(6956ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(285ns)} GenerateSessionVirtualId:988(1259ns) PrepareNewSessionHandshake:476(1285ns) {SwitchToState:178(341ns)}}}}}
2025-11-11T17:20:32.696523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518958339903350:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.707439Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903352:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.707467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518958339903352:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.708597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518958339903359:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39684
2025-11-11T17:20:32.708739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518958339903360:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39684
2025-11-11T17:20:32.714292Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518958339903350:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:32.714487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518957140988679:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(125149290ns) {Configure:104(125143391ns) {ProcessConfigured:135(48686317ns) {StartInitialHandshake:157(48683211ns) {DropHandshakes:469(2248ns) {DropIncomingHandshake:441(472ns) DropOutgoingHandshake:456(401ns)} GenerateSessionVirtualId:988(837ns) PrepareNewSessionHandshake:476(4476ns) {SwitchToState:178(2236ns)}}}}}
2025-11-11T17:20:32.685750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518952430622366:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.725724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241904:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.725781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518959720241904:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.761940Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518959720241904:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:32.716775Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518959270166575:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.719070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518957140988680:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(150634999ns) {Configure:104(150626845ns) {ProcessConfigured:135(19668706ns) {StartInitialHandshake:157(19664530ns) {DropHandshakes:469(2515ns) {DropIncomingHandshake:441(429ns) DropOutgoingHandshake:456(407ns)} GenerateSessionVirtualId:988(975ns) PrepareNewSessionHandshake:476(5598ns) {SwitchToState:178(2104ns)}}}}}
2025-11-11T17:20:32.749931Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518958339903352:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.714298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518956725589741:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.783499Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903360:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.783836Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903359:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.783897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518958339903346:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.783944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:32.784005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518958339903346:2108] [node 4] ICP22 created new session: [2:7571518958339903364:2108]
2025-11-11T17:20:32.784054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518958339903364:2108] [node 4] ICS09 handshake done sender: [2:7571518958339903350:2111] self: [2:7571518958339903349:0] peer: [4:7571518959270166576:0] socket: 464 qp: -1
2025-11-11T17:20:32.784097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518958339903364:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:32.784146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:32.785519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518958339903346:2108] [node 4] ICP18 handshake (actor [2:7571518958339903359:2115]) is too old
2025-11-11T17:20:32.785717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518958339903363:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39684
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.767552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518957140988681:2121] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(52784409ns) {Configure:104(52778961ns) {ProcessConfigured:135(96103ns) {StartInitialHandshake:157(94277ns) {DropHandshakes:469(7954ns) {DropIncomingHandshake:441(1309ns) DropOutgoingHandshake:456(383ns)} GenerateSessionVirtualId:988(1042ns) PrepareNewSessionHandshake:476(2713ns) {SwitchToState:178(1386ns)}}}}}
2025-11-11T17:20:32.717055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518959270166579:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.794930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518958339903366:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39684
2025-11-11T17:20:32.794997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518958339903347:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.795051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:32.795101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518958339903347:2109] [node 5] ICP22 created new session: [2:7571518958339903369:2109]
2025-11-11T17:20:32.795148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518958339903369:2109] [node 5] ICS09 handshake done sender: [2:7571518958339903352:2112] self: [2:7571518958339903351:0] peer: [5:7571518956725589740:0] socket: 465 qp: -1
2025-11-11T17:20:32.795215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518958339903369:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:32.795258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.717394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518959270166562:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(290261521ns) {Configure:104(290255174ns) {ProcessConfigured:135(8113831ns) {StartInitialHandshake:157(3553702ns) {DropHandshakes:469(3022ns) {DropIncomingHandshake:441(686ns) DropOutgoingHandshake:456(440ns)} GenerateSessionVirtualId:988(2987ns) PrepareNewSessionHandshake:476(2716ns) {SwitchToState:178(507ns)}}}}}
2025-11-11T17:20:32.829950Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903363:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.812991Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518956725589739:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:32.806040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518957105255788:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.838462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518957140988693:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21017
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.717478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518954975199200:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.818342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518959720241919:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43329
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.838636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518957140988694:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21017
2025-11-11T17:20:32.822329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518956725589742:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-11-11T17:20:32.822406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518956725589743:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-11-11T17:20:32.830337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518952430622366:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:32.830403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:32.830459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518952430622366:2049] [node 2] ICP22 created new session: [5:7571518956725589744:2049]
2025-11-11T17:20:32.830514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518956725589744:2049] [node 2] ICS09 handshake done sender: [5:7571518956725589739:2115] self: [5:7571518956725589740:0] peer: [2:7571518958339903351:0] socket: 468 qp: -1
2025-11-11T17:20:32.830555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518956725589744:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:32.830626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:32.874395Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518956725589743:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.821691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518957105255785:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(133769541ns) {Configure:104(133762686ns) {ProcessConfigured:135(88780ns) {StartInitialHandshake:157(85632ns) {DropHandshakes:469(3794ns) {DropIncomingHandshake:441(960ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(983ns) PrepareNewSessionHandshake:476(2068ns) {SwitchToState:178(672ns)}}}}}
2025-11-11T17:20:32.831548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518959720241895:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.838745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518957140988695:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21017
2025-11-11T17:20:32.839187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518957140988696:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21017
2025-11-11T17:20:32.839329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518957140988697:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21017
2025-11-11T17:20:32.855168Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988695:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.865859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988694:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.874113Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988697:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.874254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518952846021301:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.878432Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988693:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.774098Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518959270166570:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:32.774395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518954975199201:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:32.774450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:32.774503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518954975199201:2049] [node 2] ICP22 created new session: [4:7571518959270166583:2049]
2025-11-11T17:20:32.774547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518959270166583:2049] [node 2] ICS09 handshake done sender: [4:7571518959270166570:2115] self: [4:7571518959270166576:0] peer: [2:7571518958339903349:0] socket: 472 qp: -1
2025-11-11T17:20:32.774579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518959270166583:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:32.774619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:32.775111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518959270166582:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
2025-11-11T17:20:32.776776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518959270166575:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:32.776998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518954975199200:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:32.777038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:32.777089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518954975199200:2048] [node 1] ICP22 created new session: [4:7571518959270166585:2048]
2025-11-11T17:20:32.777130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518959270166585:2048] [node 1] ICS09 handshake done sender: [4:7571518959270166575:2117] self: [4:7571518959270166581:0] peer: [1:7571518959720241903:0] socket: 485 qp: -1
2025-11-11T17:20:32.777172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518959270166585:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:32.777202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.881883Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518956725589742:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.831636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:32.854193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518959720241895:2107] [node 4] ICP22 created new session: [1:7571518959720241920:2107]
2025-11-11T17:20:32.868338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518959720241920:2107] [node 4] ICS09 handshake done sender: [1:7571518959720241904:2114] self: [1:7571518959720241903:0] peer: [4:7571518959270166581:0] socket: 484 qp: -1
2025-11-11T17:20:32.868396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518959720241920:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:32.871538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:32.894335Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241919:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.894526Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241900:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.894554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518959720241900:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.902751Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518959720241900:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:32.869824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518954044935984:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.921710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988696:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.922310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518955425274535:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.922425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:32.922502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518955425274535:2048] [node 2] ICP22 created new session: [1:7571518959720241925:2048]
2025-11-11T17:20:32.922566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518959720241925:2048] [node 2] ICS09 handshake done sender: [1:7571518959720241900:2112] self: [1:7571518959720241899:0] peer: [2:7571518958339903372:0] socket: 481 qp: -1
2025-11-11T17:20:32.922604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518959720241925:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:32.922655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.874063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518957105255789:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:32.878435Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903366:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.926251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988685:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.926313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518957140988685:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.928546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518952846021300:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.940803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518957140988703:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21017
2025-11-11T17:20:32.941034Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518957140988685:2130] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:32.941378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518957140988679:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.941500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:32.941551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518957140988679:2108] [node 5] ICP22 created new session: [3:7571518957140988705:2108]
2025-11-11T17:20:32.941621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518957140988705:2108] [node 5] ICS09 handshake done sender: [3:7571518957140988685:2130] self: [3:7571518957140988684:0] peer: [5:7571518956725589748:0] socket: 497 qp: -1
2025-11-11T17:20:32.941664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518957140988705:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:32.941718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:32.882340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518956725589724:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.941804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518959270166580:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.941868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518959270166580:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.945498Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518959270166569:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.945553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518959270166569:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.896931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903358:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.961762Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241902:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.961810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518959720241902:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.964824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518959720241895:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(132986359ns) {HandleHandshakeStatus:329(132983752ns) {DropIncomingHandshake:441(15231ns) DropOutgoingHandshake:456(392ns) SwitchToState:178(520ns) ProcessPendingSessionEvents:495(18866165ns) {ForwardSessionEventToSession:612(18809354ns) ForwardSessionEventToSession:612(6197ns) ForwardSessionEventToSession:612(3265ns) ForwardSessionEventToSession:612(9185ns) ForwardSessionEventToSession:612(3005ns) ForwardSessionEventToSession:612(5499ns) ForwardSessionEventToSession:612(3169ns) ForwardSessionEventToSession:612(5384ns) ForwardSessionEventToSession:612(5980ns) ForwardSessionEventToSession:612(2164ns)}}}
2025-11-11T17:20:32.964989Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518959720241902:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:32.882474Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518957105255789:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.882411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518952430622367:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.957994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518959270166569:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.974519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518955425274536:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.974602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:32.974658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518955425274536:2049] [node 3] ICP22 created new session: [1:7571518959720241929:2049]
2025-11-11T17:20:32.974710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518959720241929:2049] [node 3] ICS09 handshake done sender: [1:7571518959720241902:2113] self: [1:7571518959720241901:0] peer: [3:7571518957140988702:0] socket: 483 qp: -1
2025-11-11T17:20:32.974742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518959720241929:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:32.974814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:32.896960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518958339903358:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.882599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518957105255777:2090] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.897878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518956725589735:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.988677Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241906:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.988718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518959720241906:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.989542Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518959720241916:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.989563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518959720241916:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.985572Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518957140988697:2137] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:32.991001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518957140988707:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21017
2025-11-11T17:20:32.992572Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:32.958413Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518959270166580:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:32.899355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518958339903374:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39684
2025-11-11T17:20:33.002689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518957140988679:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(46751460ns) {HandleHandshakeStatus:329(46747922ns) {DropIncomingHandshake:441(14032ns) DropOutgoingHandshake:456(857ns) SwitchToState:178(2763ns) ProcessPendingSessionEvents:495(21132005ns) {ForwardSessionEventToSession:612(17715299ns) ForwardSessionEventToSession:612(6940ns) ForwardSessionEventToSession:612(3353316ns) ForwardSessionEventToSession:612(8530ns) ForwardSessionEventToSession:612(3681ns) ForwardSessionEventToSession:612(5755ns) ForwardSessionEventToSession:612(2976ns) ForwardSessionEventToSession:612(3444ns) ForwardSessionEventToSession:612(8169ns) ForwardSessionEventToSession:612(2231ns)}}}
2025-11-11T17:20:33.002761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518952846021301:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:33.002874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:33.002928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518952846021301:2049] [node 2] ICP22 created new session: [3:7571518961435956005:2049]
2025-11-11T17:20:33.002976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518961435956005:2049] [node 2] ICS09 handshake done sender: [3:7571518957140988697:2137] self: [3:7571518957140988700:0] peer: [2:7571518958339903357:0] socket: 507 qp: -1
2025-11-11T17:20:33.003016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518961435956005:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:33.003077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:33.003770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518952846021300:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518957140988696:2136] held: no
2025-11-11T17:20:33.003943Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:33.005223Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518957140988696:2136] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.897906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518956725589735:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:32.959667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518959270166563:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.959768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:32.959825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518959270166563:2109] [node 5] ICP22 created new session: [4:7571518959270166591:2109]
2025-11-11T17:20:32.959869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518959270166591:2109] [node 5] ICS09 handshake done sender: [4:7571518959270166569:2114] self: [4:7571518959270166568:0] peer: [5:7571518956725589747:0] socket: 498 qp: -1
2025-11-11T17:20:33.001751Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518959720241906:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:33.014828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518959720241916:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:32.896609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518957105255790:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:32.906524Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518958339903354:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.959912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518959270166591:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:33.015948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518959720241894:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:33.022980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518952846021301:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:33.023202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518952846021300:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:32.898950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518956725589749:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-11-11T17:20:32.906554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518958339903354:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:33.023312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:33.023364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518952846021300:2048] [node 1] ICP22 created new session: [3:7571518961435956007:2048]
2025-11-11T17:20:33.023416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518961435956007:2048] [node 1] ICS09 handshake done sender: [3:7571518957140988696:2136] self: [3:7571518957140988702:0] peer: [1:7571518959720241901:0] socket: 482 qp: -1
2025-11-11T17:20:33.023461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518961435956007:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:32.900959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518957105255790:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.959947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:32.975794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518959270166562:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.975874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:32.975925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518959270166562:2108] [node 6] ICP22 created new session: [4:7571518959270166594:2108]
2025-11-11T17:20:32.975973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518959270166594:2108] [node 6] ICS09 handshake done sender: [4:7571518959270166580:2119] self: [4:7571518959270166578:0] peer: [6:7571518957105255793:0] socket: 509 qp: -1
2025-11-11T17:20:32.976008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518959270166594:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:32.976036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:32.976618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518959270166593:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
2025-11-11T17:20:32.991402Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518959270166593:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.991778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518959270166562:2108] [node 6] ICP18 handshake (actor [4:7571518959270166593:2121]) is too old
2025-11-11T17:20:32.991855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518959270166596:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
2025-11-11T17:20:32.901008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518957105255788:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.901274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518957105255785:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.901377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518952810288415:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.907668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518957105255792:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:32.908466Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518957105255789:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:32.908725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518957105255777:2090] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:32.908810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:32.909136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518957105255777:2090] [node 5] ICP22 created new session: [6:7571518957105255795:2090]
2025-11-11T17:20:32.909225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518957105255795:2090] [node 5] ICS09 handshake done sender: [6:7571518957105255789:2116] self: [6:7571518957105255791:0] peer: [5:7571518956725589734:0] socket: 474 qp: -1
2025-11-11T17:20:32.938029Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:33.023546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:32.899671Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518956725589735:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:33.027723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:32.899951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518956725589731:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.900001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:32.900090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518956725589731:2109] [node 6] ICP22 created new session: [5:7571518956725589751:2109]
2025-11-11T17:20:32.900145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518956725589751:2109] [node 6] ICS09 handshake done sender: [5:7571518956725589735:2114] self: [5:7571518956725589734:0] peer: [6:7571518957105255791:0] socket: 508 qp: -1
2025-11-11T17:20:32.900182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518956725589751:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:32.900247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:32.958600Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518956725589749:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.959254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518952430622365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.959367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518956725589753:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-11-11T17:20:32.982951Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4958) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:32.984460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518956725589754:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-11-11T17:20:33.000437Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:33.009543Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518956725589742:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:33.013966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518956725589724:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4958) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518956725589743:2118] held: no
2025-11-11T17:20:33.014138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518952430622367:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518956725589742:2117] held: no
2025-11-11T17:20:33.014426Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518956725589743:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:33.029978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518952430622367:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:33.030066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:33.030133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518952430622367:2050] [node 3] ICP22 created new session: [5:7571518961020557053:2050]
2025-11-11T17:20:33.030184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518961020557053:2050] [node 3] ICS09 handshake done sender: [5:7571518956725589742:2117] self: [5:7571518956725589748:0] peer: [3:7571518957140988684:0] socket: 504 qp: -1
2025-11-11T17:20:33.030226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518961020557053:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:33.030266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:33.030697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518956725589724:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:33.030761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:33.030805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518956725589724:2108] [node 4] ICP22 created new session: [5:7571518961020557055:2108]
2025-11-11T17:20:33.030847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518961020557055:2108] [node 4] ICS09 handshake done sender: [5:7571518956725589743:2118] self: [5:7571518956725589747:0] peer: [4:7571518959270166568:0] socket: 506 qp: -1
2025-11-11T17:20:33.032392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518961020557055:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:33.049627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:32.909273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518957105255795:2090] [node 5] ICS10 traffic start
2025-11-11T17:20:33.046585Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518959270166596:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:33.050964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518954975199202:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.941962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518958339903363:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:32.957849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518954044935984:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518958339903363:2117] held: no
2025-11-11T17:20:32.963222Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518958339903358:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:32.966682Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518958339903354:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:32.978003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518954044935984:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:32.978053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:32.978102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518954044935984:2048] [node 1] ICP22 created new session: [2:7571518958339903377:2048]
2025-11-11T17:20:32.978182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518958339903377:2048] [node 1] ICS09 handshake done sender: [2:7571518958339903363:2117] self: [2:7571518958339903372:0] peer: [1:7571518959720241899:0] socket: 487 qp: -1
2025-11-11T17:20:32.978216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518958339903377:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:32.978274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:33.031085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518959720241896:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.978637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518954044935985:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.978682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:33.031150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:33.031192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518959720241896:2109] [node 6] ICP22 created new session: [1:7571518964015209230:2109]
2025-11-11T17:20:32.978720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518954044935985:2049] [node 3] ICP22 created new session: [2:7571518958339903379:2049]
2025-11-11T17:20:33.031229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518964015209230:2109] [node 6] ICS09 handshake done sender: [1:7571518959720241916:2118] self: [1:7571518959720241915:0] peer: [6:7571518957105255801:0] socket: 467 qp: -1
2025-11-11T17:20:32.978756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518958339903379:2049] [node 3] ICS09 handshake done sender: [2:7571518958339903358:2114] self: [2:7571518958339903357:0] peer: [3:7571518957140988700:0] socket: 500 qp: -1
2025-11-11T17:20:33.031275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518964015209230:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:32.978783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518958339903379:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:33.031332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:33.032889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518959720241894:2108] [node 5] ICP22 created new session: [1:7571518964015209229:2108]
2025-11-11T17:20:32.978810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:33.050944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518964015209229:2108] [node 5] ICS09 handshake done sender: [1:7571518959720241906:2115] self: [1:7571518959720241905:0] peer: [5:7571518956725589755:0] socket: 480 qp: -1
2025-11-11T17:20:32.981714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518958339903348:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:32.981763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:32.981810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518958339903348:2110] [node 6] ICP22 created new session: [2:7571518958339903381:2110]
2025-11-11T17:20:32.981859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518958339903381:2110] [node 6] ICS09 handshake done sender: [2:7571518958339903354:2113] self: [2:7571518958339903353:0] peer: [6:7571518957105255794:0] socket: 499 qp: -1
2025-11-11T17:20:32.981887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518958339903381:2110] [node 6] ICS10 traffic start
2025-11-11T17:20:32.981924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:33.069725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988687:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:33.054531Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:33.055847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518956725589724:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(20621788ns) {HandleHandshakeStatus:329(20619466ns) {DropIncomingHandshake:441(318ns) DropOutgoingHandshake:456(169ns) SwitchToState:178(138ns) ProcessPendingSessionEvents:495(1028ns)}}
2025-11-11T17:20:33.056074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518952430622365:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518956725589749:2119] held: no
2025-11-11T17:20:33.054388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518964015209229:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:33.074073Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:33.075413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518963565133894:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24083
2025-11-11T17:20:32.909317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:33.069784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518957140988687:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:33.070947Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518957140988687:2131] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:33.071241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518957140988680:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:33.071350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:33.071406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518957140988680:2107] [node 4] ICP22 created new session: [3:7571518961435956011:2107]
2025-11-11T17:20:33.071449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518961435956011:2107] [node 4] ICS09 handshake done sender: [3:7571518957140988687:2131] self: [3:7571518957140988686:0] peer: [4:7571518963565133893:0] socket: 528 qp: -1
2025-11-11T17:20:33.071482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518961435956011:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:33.071538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:33.054444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:33.084304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518954975199202:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518959270166596:2122] held: no
2025-11-11T17:20:32.964075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518957105255797:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:32.964194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518957105255798:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:32.964272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518957105255799:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:32.980571Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518957105255797:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:32.980812Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518957105255790:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:32.981250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518952810288414:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:32.981340Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518957105255788:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:32.986180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518952810288415:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:32.986265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:32.986314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518952810288415:2049] [node 2] ICP22 created new session: [6:7571518957105255802:2049]
2025-11-11T17:20:32.986367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518957105255802:2049] [node 2] ICS09 handshake done sender: [6:7571518957105255788:2115] self: [6:7571518957105255794:0] peer: [2:7571518958339903353:0] socket: 503 qp: -1
2025-11-11T17:20:32.986397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518957105255802:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:32.986429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:32.998161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518957105255785:2094] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:32.998254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:32.998313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518957105255785:2094] [node 4] ICP22 created new session: [6:7571518957105255804:2094]
2025-11-11T17:20:32.998368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518957105255804:2094] [node 4] ICS09 handshake done sender: [6:7571518957105255790:2117] self: [6:7571518957105255793:0] peer: [4:7571518959270166578:0] socket: 510 qp: -1
2025-11-11T17:20:32.998409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518957105255804:2094] [node 4] ICS10 traffic start
2025-11-11T17:20:32.998472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:33.009106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518961400223102:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:33.015731Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518957105255797:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:33.027822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518961400223103:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:33.033091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518952810288414:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:33.056721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518961020557052:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-11-11T17:20:33.091184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518959720241894:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(72030948ns) {HandleHandshakeStatus:329(72028659ns) {DropIncomingHandshake:441(14180ns) DropOutgoingHandshake:456(311ns) SwitchToState:178(698ns) ProcessPendingSessionEvents:495(33399672ns) {ForwardSessionEventToSession:612(11248965ns) ForwardSessionEventToSession:612(6796693ns) ForwardSessionEventToSession:612(1161376ns) ForwardSessionEventToSession:612(8043ns) ForwardSessionEventToSession:612(4092249ns) ForwardSessionEventToSession:612(4177ns) ForwardSessionEventToSession:612(5581537ns) ForwardSessionEventToSession:612(9269ns) ForwardSessionEventToSession:612(4980ns) ForwardSessionEventToSession:612(4461188ns)}}}
2025-11-11T17:20:33.038036Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:33.050937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:33.054898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518952810288414:2048] [node 1] ICP22 created new session: [6:7571518961400223104:2048]
2025-11-11T17:20:33.056856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518961400223104:2048] [node 1] ICS09 handshake done sender: [6:7571518957105255797:2119] self: [6:7571518957105255801:0] peer: [1:7571518959720241915:0] socket: 521 qp: -1
2025-11-11T17:20:33.072576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518961400223104:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:33.073730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:33.087130Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518961400223103:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:33.087489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518952810288416:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:33.090320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518952810288414:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(56997298ns) {HandleHandshakeStatus:329(56994515ns) {DropIncomingHandshake:441(780ns) DropOutgoingHandshake:456(5463ns) SwitchToState:178(515ns) ProcessPendingSessionEvents:495(2167507ns) {ForwardSessionEventToSession:612(2108752ns) ForwardSessionEventToSession:612(10044ns) ForwardSessionEventToSession:612(3933ns) ForwardSessionEventToSession:612(3139ns) ForwardSessionEventToSession:612(5219ns) ForwardSessionEventToSession:612(3296ns) ForwardSessionEventToSession:612(5688ns) ForwardSessionEventToSession:612(9574ns) ForwardSessionEventToSession:612(2574ns) ForwardSessionEventToSession:612(4660ns)}}}
2025-11-11T17:20:33.090408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571518952810288414:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:33.075663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518956725589749:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:33.089727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518952430622365:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:33.104450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:33.104564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518952430622365:2048] [node 1] ICP22 created new session: [5:7571518961020557057:2048]
2025-11-11T17:20:33.104627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518961020557057:2048] [node 1] ICS09 handshake done sender: [5:7571518956725589749:2119] self: [5:7571518956725589755:0] peer: [1:7571518959720241905:0] socket: 513 qp: -1
2025-11-11T17:20:33.104670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518961020557057:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:33.104722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:33.105067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518952430622365:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23053883ns) {HandleHandshakeStatus:329(23050682ns) {DropIncomingHandshake:441(3113ns) DropOutgoingHandshake:456(569ns) SwitchToState:178(1019ns) ProcessPendingSessionEvents:495(1115ns)}}
2025-11-11T17:20:33.092087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518957140988689:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:33.092138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518957140988689:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:33.121810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518954975199202:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(22614435ns) {HandleHandshakeStatus:395(22608010ns) {LogHandshakeFail:472(17264ns) {UpdateErrorStateLog:544(6264ns)} DropOutgoingHandshake:456(3412ns) ProcessPendingSessionEvents:495(13511089ns) {DropSessionEvent:512(14881ns) DropSessionEvent:512(4721ns) DropSessionEvent:512(4583ns) DropSessionEvent:512(4828ns) DropSessionEvent:512(13424829ns) DropSessionEvent:512(26751ns) DropSessionEvent:512(4966ns) DropSessionEvent:512(5134ns) DropSessionEvent:512(4255ns) DropSessionEvent:512(4013ns)}}}
2025-11-11T17:20:33.122007Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518959270166596:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:33.131657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518954975199202:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:33.131759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:33.131819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518954975199202:2050] [node 3] ICP22 created new session: [4:7571518963565133895:2050]
2025-11-11T17:20:33.131875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518963565133895:2050] [node 3] ICS09 handshake done sender: [4:7571518959270166596:2122] self: [4:7571518963565133893:0] peer: [3:7571518957140988686:0] socket: 505 qp: -1
2025-11-11T17:20:33.131914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518963565133895:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:33.131966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:33.138345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518961400223107:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25929
2025-11-11T17:20:33.138870Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518961400223103:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:33.139095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518952810288416:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:33.138025Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518957140988689:2132] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:33.139189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:33.139240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518952810288416:2050] [node 3] ICP22 created new session: [6:7571518961400223108:2050]
2025-11-11T17:20:33.139317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518961400223108:2050] [node 3] ICS09 handshake done sender: [6:7571518961400223103:2123] self: [6:7571518961400223106:0] peer: [3:7571518957140988688:0] socket: 477 qp: -1
2025-11-11T17:20:33.139360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518961400223108:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:33.139433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:33.146093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518957140988681:2121] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:33.146215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:33.146272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518957140988681:2121] [node 6] ICP22 created new session: [3:7571518961435956014:2121]
2025-11-11T17:20:33.146329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518961435956014:2121] [node 6] ICS09 handshake done sender: [3:7571518957140988689:2132] self: [3:7571518957140988688:0] peer: [6:7571518961400223106:0] socket: 476 qp: -1
2025-11-11T17:20:33.146371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518961435956014:2121] [node 6] ICS10 traffic start
2025-11-11T17:20:33.146437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:33.177424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:33.177764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518961435956007:2048] [node 1] ICS01 socket: 482 reason# EndOfStream
2025-11-11T17:20:33.177730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:33.177958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518958339903377:2048] [node 1] ICS01 socket: 487 reason# EndOfStream
2025-11-11T17:20:33.177812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518952846021300:2048] [node 1] ICP30 unregister session Session# [3:7571518961435956007:2048] VirtualId# [3:7571518957140988702:0]
2025-11-11T17:20:33.177997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518954044935984:2048] [node 1] ICP30 unregister session Session# [2:7571518958339903377:2048] VirtualId# [2:7571518958339903372:0]
2025-11-11T17:20:33.177845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518961435956007:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.178028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518958339903377:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.177940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:33.178102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:33.177985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:33.178147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:33.181890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:33.182113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518961020557057:2048] [node 1] ICS01 socket: 513 reason# EndOfStream
2025-11-11T17:20:33.182152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518952430622365:2048] [node 1] ICP30 unregister session Session# [5:7571518961020557057:2048] VirtualId# [5:7571518956725589755:0]
2025-11-11T17:20:33.182188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518961020557057:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.182269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:33.182320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:33.181763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:33.182064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518961400223104:2048] [node 1] ICS07 socket disconnect 521 reason# EndOfStream
2025-11-11T17:20:33.182092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518961400223104:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.182195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:33.182238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518961400223104:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:33.183206Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Couldn't connect to any resolved address
2025-11-11T17:20:33.182991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:33.183246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518959270166585:2048] [node 1] ICS01 socket: 485 reason# EndOfStream
2025-11-11T17:20:33.183288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518954975199200:2048] [node 1] ICP30 unregister session Session# [4:7571518959270166585:2048] VirtualId# [4:7571518959270166581:0]
2025-11-11T17:20:33.183323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518959270166585:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.183424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:33.183474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:33.183557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518952810288414:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:33.183616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518961400223104:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:33.183651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518952810288414:2048] [node 1] ICP30 unregister session Session# [6:7571518961400223104:2048] VirtualId# [6:7571518957105255801:0]
2025-11-11T17:20:33.183695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:33.183727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518952810288414:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:33.183740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:33.194708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518952810288414:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:33.248751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:33.248985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518961435956005:2049] [node 2] ICS01 socket: 507 reason# EndOfStream
2025-11-11T17:20:33.249022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518952846021301:2049] [node 2] ICP30 unregister session Session# [3:7571518961435956005:2049] VirtualId# [3:7571518957140988700:0]
2025-11-11T17:20:33.249060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518961435956005:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.249161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:33.249207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:33.258547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:33.258722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518956725589744:2049] [node 2] ICS01 socket: 468 reason# EndOfStream
2025-11-11T17:20:33.258760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518952430622366:2049] [node 2] ICP30 unregister session Session# [5:7571518956725589744:2049] VirtualId# [5:7571518956725589740:0]
2025-11-11T17:20:33.258785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518956725589744:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.258843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:33.258886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:33.258977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:33.259203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518957105255802:2049] [node 2] ICS01 socket: 503 reason# EndOfStream
2025-11-11T17:20:33.259238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518952810288415:2049] [node 2] ICP30 unregister session Session# [6:7571518957105255802:2049] VirtualId# [6:7571518957105255794:0]
2025-11-11T17:20:33.259268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518957105255802:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.259342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:33.259380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:33.257891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:33.258149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518959270166583:2049] [node 2] ICS01 socket: 472 reason# EndOfStream
2025-11-11T17:20:33.258188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518954975199201:2049] [node 2] ICP30 unregister session Session# [4:7571518959270166583:2049] VirtualId# [4:7571518959270166576:0]
2025-11-11T17:20:33.258228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518959270166583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.258382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:33.258435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:33.321790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:33.322049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518963565133895:2050] [node 3] ICS01 socket: 505 reason# EndOfStream
2025-11-11T17:20:33.322084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518954975199202:2050] [node 3] ICP30 unregister session Session# [4:7571518963565133895:2050] VirtualId# [4:7571518963565133893:0]
2025-11-11T17:20:33.322121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518963565133895:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.322215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:33.322268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:33.323145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:33.323365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518961020557053:2050] [node 3] ICS01 socket: 504 reason# EndOfStream
2025-11-11T17:20:33.323410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518952430622367:2050] [node 3] ICP30 unregister session Session# [5:7571518961020557053:2050] VirtualId# [5:7571518956725589748:0]
2025-11-11T17:20:33.323441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518961020557053:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.323535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:33.323574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:33.328878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:33.329176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518961400223108:2050] [node 3] ICS01 socket: 477 reason# EndOfStream
2025-11-11T17:20:33.329234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518952810288416:2050] [node 3] ICP30 unregister session Session# [6:7571518961400223108:2050] VirtualId# [6:7571518961400223106:0]
2025-11-11T17:20:33.329275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518961400223108:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.329405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:33.329466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:33.377793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:33.378056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518957105255804:2094] [node 4] ICS01 socket: 510 reason# EndOfStream
2025-11-11T17:20:33.378100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518957105255785:2094] [node 4] ICP30 unregister session Session# [6:7571518957105255804:2094] VirtualId# [6:7571518957105255793:0]
2025-11-11T17:20:33.378145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518957105255804:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.378235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:33.378280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:33.378378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:33.378582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518961020557055:2108] [node 4] ICS01 socket: 506 reason# EndOfStream
2025-11-11T17:20:33.378617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518956725589724:2108] [node 4] ICP30 unregister session Session# [5:7571518961020557055:2108] VirtualId# [5:7571518956725589747:0]
2025-11-11T17:20:33.378650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518961020557055:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.378699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:33.378738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:33.433775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:33.434047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518957105255795:2090] [node 5] ICS01 socket: 474 reason# EndOfStream
2025-11-11T17:20:33.434088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518957105255777:2090] [node 5] ICP30 unregister session Session# [6:7571518957105255795:2090] VirtualId# [6:7571518957105255791:0]
2025-11-11T17:20:33.434122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518957105255795:2090] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:33.434226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:33.434263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:33.544945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518960807688575:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:33.544993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518960807688576:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:33.579843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518963938761776:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:33.579896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518963938761777:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:33.603484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518961679081338:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:33.603543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518961679081339:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:33.624045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518963674081605:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:33.624084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518963674081606:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:33.624112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518963674081607:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:33.655561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518961749584789:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:33.655624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518961749584790:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:33.655652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518961749584791:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:33.671853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518962052850686:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:33.671889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518962052850687:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:33.671914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518962052850688:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:34.054351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518966347818008:2071] [node 5] ICP01 ready to work
2025-11-11T17:20:34.072752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518966044552128:2090] [node 4] ICP01 ready to work
2025-11-11T17:20:34.189887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518967969048964:2100] [node 5] ICP01 ready to work
2025-11-11T17:20:34.197685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518968233729159:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:34.209700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518965974048698:2109] [node 4] ICP01 ready to work
2025-11-11T17:20:34.226324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518963674081607:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(34230021ns) {Configure:104(34226456ns) {ProcessConfigured:135(68514ns) {StartInitialHandshake:157(65580ns) {DropHandshakes:469(2202ns) {DropIncomingHandshake:441(687ns) DropOutgoingHandshake:456(367ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:476(1381ns) {SwitchToState:178(592ns)}}}}}
2025-11-11T17:20:34.223359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518968233729160:2133] [node 6] ICP01 ready to work
2025-11-11T17:20:34.223670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518965102655934:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:34.214434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518961749584790:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(160867452ns) {Configure:104(160862025ns) {ProcessConfigured:135(82603ns) {StartInitialHandshake:157(79421ns) {DropHandshakes:469(1954ns) {DropIncomingHandshake:441(414ns) DropOutgoingHandshake:456(403ns)} GenerateSessionVirtualId:988(634ns) PrepareNewSessionHandshake:476(1875ns) {SwitchToState:178(615ns)}}}}}
2025-11-11T17:20:34.223205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518966044552149:2091] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.245761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518965102655935:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:34.262166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518965102655937:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:34.262629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518965102655938:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35614
2025-11-11T17:20:34.253395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518965974048699:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:34.234359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518966347818035:2078] [node 4] ICP01 ready to work
2025-11-11T17:20:34.253472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518965974048700:2107] [node 5] ICP01 ready to work
2025-11-11T17:20:34.309812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518963938761777:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(185227275ns) {Configure:104(185221246ns) {ProcessConfigured:135(21544286ns) {StartInitialHandshake:157(21542049ns) {DropHandshakes:469(2689ns) {DropIncomingHandshake:441(466ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:476(4209ns) {SwitchToState:178(2003ns)}}}}}
2025-11-11T17:20:34.302641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518967969048974:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:34.275633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518962052850686:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(64726054ns) {Configure:104(64717093ns) {ProcessConfigured:135(67742ns) {StartInitialHandshake:157(65219ns) {DropHandshakes:469(2831ns) {DropIncomingHandshake:441(565ns) DropOutgoingHandshake:456(538ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:476(1823ns) {SwitchToState:178(711ns)}}}}}
2025-11-11T17:20:34.289952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518960807688576:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(242042913ns) {Configure:104(242037818ns) {ProcessConfigured:135(35709778ns) {StartInitialHandshake:157(35707448ns) {DropHandshakes:469(2483ns) {DropIncomingHandshake:441(500ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(574ns) PrepareNewSessionHandshake:476(5705ns) {SwitchToState:178(2300ns)}}}}}
2025-11-11T17:20:34.315609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518968233729163:2121] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.336537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518961749584791:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(121756654ns) {Configure:104(121752673ns) {ProcessConfigured:135(22663363ns) {StartInitialHandshake:157(22660335ns) {DropHandshakes:469(1268ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(269ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:476(4419ns) {SwitchToState:178(1928ns)}}}}}
2025-11-11T17:20:34.339018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552158:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.339841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518966044552128:2090] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(220942099ns) {Configure:104(220936292ns) {ProcessConfigured:135(36001691ns) {StartInitialHandshake:157(35998843ns) {DropHandshakes:469(2372ns) {DropIncomingHandshake:441(443ns) DropOutgoingHandshake:456(543ns)} GenerateSessionVirtualId:988(1408ns) PrepareNewSessionHandshake:476(2488ns) {SwitchToState:178(1345ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.357626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518967969048977:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18008
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.365360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518968233729168:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42812
2025-11-11T17:20:34.365447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518968233729169:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42812
2025-11-11T17:20:34.365507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518968233729170:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42812
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.377913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518966044552158:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.378072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518966044552128:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.377992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518965102655934:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(104000171ns) {Configure:104(103994895ns) {ProcessConfigured:135(68829ns) {StartInitialHandshake:157(66049ns) {DropHandshakes:469(2660ns) {DropIncomingHandshake:441(646ns) DropOutgoingHandshake:456(483ns)} GenerateSessionVirtualId:988(735ns) PrepareNewSessionHandshake:476(2170ns) {SwitchToState:178(623ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.380861Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655938:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.381297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518965102655947:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35614
2025-11-11T17:20:34.381411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518965102655950:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35614
2025-11-11T17:20:34.366752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518967969048978:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18008
2025-11-11T17:20:34.399312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518968233729163:2121] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(67509464ns) {Configure:104(67505306ns) {ProcessConfigured:135(23523600ns) {StartInitialHandshake:157(23520760ns) {DropHandshakes:469(2821ns) {DropIncomingHandshake:441(644ns) DropOutgoingHandshake:456(436ns)} GenerateSessionVirtualId:988(1268ns) PrepareNewSessionHandshake:476(3293ns) {SwitchToState:178(1752ns)}}}}}
2025-11-11T17:20:34.370998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818058:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.404328Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518967969048978:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.405090Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518967969048970:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.405136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518967969048970:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.408246Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518967969048977:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.415575Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655943:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.415614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518965102655943:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.409991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729170:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.420119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518965102655935:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(142062860ns) {Configure:104(142058223ns) {ProcessConfigured:135(2176031ns) {StartInitialHandshake:157(2172972ns) {DropHandshakes:469(6937ns) {DropIncomingHandshake:441(626ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:476(3052ns) {SwitchToState:178(990ns)}}}}}
2025-11-11T17:20:34.420931Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518965102655943:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:34.381886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518966347818008:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(278624191ns) {Configure:104(278620761ns) {ProcessConfigured:135(30332081ns) {StartInitialHandshake:157(30329209ns) {DropHandshakes:469(1083ns) {DropIncomingHandshake:441(259ns) DropOutgoingHandshake:456(257ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:476(4112ns) {SwitchToState:178(1975ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.432594Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729169:2138] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.416894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518963674081606:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(258466023ns) {Configure:104(251007634ns) {ProcessConfigured:135(7449963ns) {StartInitialHandshake:157(7446798ns) {DropHandshakes:469(2671ns) {DropIncomingHandshake:441(685ns) DropOutgoingHandshake:456(471ns)} GenerateSessionVirtualId:988(389ns) PrepareNewSessionHandshake:476(2573ns) {SwitchToState:178(999ns)}}}}}
2025-11-11T17:20:34.417063Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518967969048970:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:34.442317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552165:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.404720Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518966347818058:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.433830Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655950:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.414013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518962052850686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.448443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518965974048701:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4473
2025-11-11T17:20:34.437778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729168:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.458933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552166:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.421470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518966347818035:2078] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(183831291ns) {Configure:104(183823425ns) {ProcessConfigured:135(3396087ns) {StartInitialHandshake:157(3393481ns) {DropHandshakes:469(1782ns) {DropIncomingHandshake:441(692ns) DropOutgoingHandshake:456(249ns)} GenerateSessionVirtualId:988(2730ns) PrepareNewSessionHandshake:476(2753ns) {SwitchToState:178(1193ns)}}}}}
2025-11-11T17:20:34.458994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552167:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.485937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518961749584789:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(182811077ns) {Configure:104(182804716ns) {ProcessConfigured:135(11705821ns) {StartInitialHandshake:157(11703597ns) {DropHandshakes:469(2611ns) {DropIncomingHandshake:441(571ns) DropOutgoingHandshake:456(411ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:476(5373ns) {SwitchToState:178(2721ns)}}}}}
2025-11-11T17:20:34.421903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818062:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.422330Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518966347818058:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:34.430194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518962052850686:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.430263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:34.430311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518962052850686:2048] [node 1] ICP22 created new session: [6:7571518966347818063:2048]
2025-11-11T17:20:34.430357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518966347818063:2048] [node 1] ICS09 handshake done sender: [6:7571518966347818058:2114] self: [6:7571518966347818059:0] peer: [1:7571518965102655942:0] socket: 487 qp: -1
2025-11-11T17:20:34.430403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518966347818063:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:34.430437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:34.432439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518963674081606:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.433979Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655947:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.490662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518967969048964:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.502036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:34.502095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518967969048964:2100] [node 5] ICP22 created new session: [4:7571518967969048985:2100]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.530814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818067:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.533916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818066:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.459717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518965102655957:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35614
2025-11-11T17:20:34.490509Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729165:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.448628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518965974048702:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4473
2025-11-11T17:20:34.448917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518965974048710:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4473
2025-11-11T17:20:34.449011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518965974048711:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4473
2025-11-11T17:20:34.449157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518965974048712:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4473
2025-11-11T17:20:34.462120Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048702:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.462253Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048711:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.470054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518965974048698:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(168537161ns) {Configure:104(168532922ns) {ProcessConfigured:135(55977ns) {StartInitialHandshake:157(53509ns) {DropHandshakes:469(2484ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(347ns)} GenerateSessionVirtualId:988(926ns) PrepareNewSessionHandshake:476(1374ns) {SwitchToState:178(532ns)}}}}}
2025-11-11T17:20:34.471787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048701:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.478042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518961679081338:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.510347Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048712:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.526759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518965974048722:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4473
2025-11-11T17:20:34.527449Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048710:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.498087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518965102655937:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.498207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:34.498274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518965102655937:2109] [node 6] ICP22 created new session: [1:7571518965102655958:2109]
2025-11-11T17:20:34.498337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518965102655958:2109] [node 6] ICS09 handshake done sender: [1:7571518965102655943:2113] self: [1:7571518965102655942:0] peer: [6:7571518966347818059:0] socket: 476 qp: -1
2025-11-11T17:20:34.498380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518965102655958:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:34.498436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:34.525929Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22598) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:34.524675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518967969048986:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.490557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518968233729165:2135] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.575463Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518966347818067:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.565514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518961679081339:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(240078388ns) {Configure:104(235729162ns) {ProcessConfigured:135(31133878ns) {StartInitialHandshake:157(31131581ns) {DropHandshakes:469(3510ns) {DropIncomingHandshake:441(663ns) DropOutgoingHandshake:456(597ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:476(3847ns) {SwitchToState:178(1091ns)}}}}}
2025-11-11T17:20:34.514102Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655939:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.546833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518966044552128:2090] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22598) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518966044552158:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.497758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518968233729163:2121] [node 5] ICP35 event processing took too much time PendingConnection:278(29070200ns) {IncomingHandshake:261(18855899ns) {DropIncomingHandshake:441(142ns) PrepareNewSessionHandshake:476(2273ns) {SwitchToState:178(936ns)} GenerateSessionVirtualId:988(233ns) IssueIncomingHandshakeReply:193(9816ns)}}
2025-11-11T17:20:34.593101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518961679081339:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.586483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518966347818035:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.525333Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518967969048978:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:34.565410Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518966044552158:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:34.514148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518965102655939:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.594441Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518966347818066:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.612149Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7819) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:34.613704Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048709:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.613746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518965974048709:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.506834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518963938761776:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.620591Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518965974048702:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:34.525526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518963674081606:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.522613Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655957:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.591929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518966044552166:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.626187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518961679081338:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7819) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518965974048702:2112] held: no
2025-11-11T17:20:34.626317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518961679081338:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.626386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:34.626438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518961679081338:2048] [node 1] ICP22 created new session: [3:7571518965974048725:2048]
2025-11-11T17:20:34.626496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518965974048725:2048] [node 1] ICS09 handshake done sender: [3:7571518965974048702:2112] self: [3:7571518965974048719:0] peer: [1:7571518965102655936:0] socket: 477 qp: -1
2025-11-11T17:20:34.626535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518965974048725:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:34.626571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:34.632485Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048704:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.632525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518965974048704:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.594589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518962052850688:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.525576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:34.507253Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518968233729165:2135] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.592032Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518966044552167:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.522721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655941:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.611022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818071:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.507603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518968233729159:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.642364Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518965974048709:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:34.525634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518963674081606:2049] [node 2] ICP22 created new session: [4:7571518967969048987:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.592161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.522744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518965102655941:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.524270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518965102655939:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:34.533691Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518965102655941:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:34.555230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518960807688576:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.555314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:34.555376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518960807688576:2049] [node 3] ICP22 created new session: [1:7571518965102655964:2049]
2025-11-11T17:20:34.555441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518965102655964:2049] [node 3] ICS09 handshake done sender: [1:7571518965102655939:2111] self: [1:7571518965102655936:0] peer: [3:7571518965974048719:0] socket: 474 qp: -1
2025-11-11T17:20:34.555482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518965102655964:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:34.555570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:34.563796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518960807688575:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.563882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:34.563919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518960807688575:2048] [node 2] ICP22 created new session: [1:7571518965102655966:2048]
2025-11-11T17:20:34.563965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518965102655966:2048] [node 2] ICS09 handshake done sender: [1:7571518965102655941:2112] self: [1:7571518965102655940:0] peer: [2:7571518968233729178:0] socket: 475 qp: -1
2025-11-11T17:20:34.563994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518965102655966:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:34.564019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:34.611259Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655949:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.611285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518965102655949:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.616187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518965102655949:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:34.629077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518965102655953:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.629105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518965102655953:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.633214Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518965102655953:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:34.633279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518965102655934:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.633335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:34.633385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518965102655934:2108] [node 5] ICP22 created new session: [1:7571518965102655971:2108]
2025-11-11T17:20:34.633429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518965102655971:2108] [node 5] ICS09 handshake done sender: [1:7571518965102655949:2115] self: [1:7571518965102655948:0] peer: [5:7571518966044552173:0] socket: 514 qp: -1
2025-11-11T17:20:34.633465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518965102655971:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:34.633505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:34.634819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518965102655935:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.634892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:34.634936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518965102655935:2107] [node 4] ICP22 created new session: [1:7571518965102655973:2107]
2025-11-11T17:20:34.634981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518965102655973:2107] [node 4] ICS09 handshake done sender: [1:7571518965102655953:2117] self: [1:7571518965102655952:0] peer: [4:7571518967969048993:0] socket: 482 qp: -1
2025-11-11T17:20:34.635015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518965102655973:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:34.635043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:34.507646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:34.621787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518966347818071:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.525676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518967969048987:2049] [node 2] ICS09 handshake done sender: [4:7571518967969048978:2115] self: [4:7571518967969048983:0] peer: [2:7571518968233729164:0] socket: 493 qp: -1
2025-11-11T17:20:34.666687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518965974048700:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.666786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:34.592316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.592699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518961749584791:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.610147Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518966044552169:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.610324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518961749584789:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.612520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518966044552128:2090] [node 4] ICP35 event processing took too much time PendingConnection:278(19790740ns) {HandleHandshakeStatus:395(19787583ns) {LogHandshakeFail:472(18241ns) {UpdateErrorStateLog:544(4781ns)} DropOutgoingHandshake:456(1243ns) ProcessPendingSessionEvents:495(18739514ns) {DropSessionEvent:512(11109ns) DropSessionEvent:512(6782ns) DropSessionEvent:512(4526ns) DropSessionEvent:512(4082ns) DropSessionEvent:512(4752ns) DropSessionEvent:512(10916127ns) DropSessionEvent:512(7767641ns) DropSessionEvent:512(6198ns) DropSessionEvent:512(4572ns) DropSessionEvent:512(5177ns)}}}
2025-11-11T17:20:34.612814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518966044552128:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.632697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:34.632766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552174:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.634410Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518966044552169:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:34.639499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518961749584789:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.639569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:34.639612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518961749584789:2048] [node 1] ICP22 created new session: [5:7571518966044552176:2048]
2025-11-11T17:20:34.639658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518966044552176:2048] [node 1] ICS09 handshake done sender: [5:7571518966044552169:2120] self: [5:7571518966044552173:0] peer: [1:7571518965102655948:0] socket: 518 qp: -1
2025-11-11T17:20:34.639695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518966044552176:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:34.639750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:34.640436Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518966044552157:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.640460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518966044552157:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.642680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518966044552128:2090] [node 4] ICP22 created new session: [5:7571518966044552175:2090]
2025-11-11T17:20:34.643065Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518966044552157:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:34.647869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518966044552175:2090] [node 4] ICS09 handshake done sender: [5:7571518966044552158:2114] self: [5:7571518966044552162:0] peer: [4:7571518967969048969:0] socket: 480 qp: -1
2025-11-11T17:20:34.652605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518966044552175:2090] [node 4] ICS10 traffic start
2025-11-11T17:20:34.652650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:34.652793Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518966044552168:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.659033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518966044552149:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.659115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:34.659166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518966044552149:2091] [node 6] ICP22 created new session: [5:7571518966044552181:2091]
2025-11-11T17:20:34.659216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518966044552181:2091] [node 6] ICS09 handshake done sender: [5:7571518966044552157:2113] self: [5:7571518966044552156:0] peer: [6:7571518966347818072:0] socket: 519 qp: -1
2025-11-11T17:20:34.659251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518966044552181:2091] [node 6] ICS10 traffic start
2025-11-11T17:20:34.659309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:34.660965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518961749584790:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.661148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552179:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.674167Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518966044552167:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:34.677876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518966044552128:2090] [node 4] ICP35 event processing took too much time PendingConnection:278(45136201ns) {HandleHandshakeStatus:329(45134034ns) {DropIncomingHandshake:441(804ns) DropOutgoingHandshake:456(580ns) SwitchToState:178(376ns) ProcessPendingSessionEvents:495(846ns)}}
2025-11-11T17:20:34.677937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518961749584791:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.678032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:34.678089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518961749584791:2050] [node 3] ICP22 created new session: [5:7571518966044552184:2050]
2025-11-11T17:20:34.678134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518966044552184:2050] [node 3] ICS09 handshake done sender: [5:7571518966044552167:2118] self: [5:7571518966044552172:0] peer: [3:7571518965974048708:0] socket: 507 qp: -1
2025-11-11T17:20:34.678183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518966044552184:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:34.678222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.507712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518968233729159:2107] [node 4] ICP22 created new session: [2:7571518968233729180:2107]
2025-11-11T17:20:34.527364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518967969048985:2100] [node 5] ICS09 handshake done sender: [4:7571518967969048970:2112] self: [4:7571518967969048969:0] peer: [5:7571518966044552162:0] socket: 470 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.666851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518965974048700:2107] [node 5] ICP22 created new session: [3:7571518965974048729:2107]
2025-11-11T17:20:34.631959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818070:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.507778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518968233729180:2107] [node 4] ICS09 handshake done sender: [2:7571518968233729165:2135] self: [2:7571518968233729164:0] peer: [4:7571518967969048983:0] socket: 491 qp: -1
2025-11-11T17:20:34.529917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518967969048987:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:34.666912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518965974048729:2107] [node 5] ICS09 handshake done sender: [3:7571518965974048709:2115] self: [3:7571518965974048708:0] peer: [5:7571518966044552172:0] socket: 505 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.635890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518966347818008:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.691609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518966044552186:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2922
2025-11-11T17:20:34.533079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:34.507818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518968233729180:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:34.636798Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518966347818067:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:34.647581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518966347818035:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.649354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818073:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.649404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818074:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.649924Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518966347818071:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:34.650148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518966347818008:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.650201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:34.650247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518966347818008:2071] [node 5] ICP22 created new session: [6:7571518966347818075:2071]
2025-11-11T17:20:34.650294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518966347818075:2071] [node 5] ICS09 handshake done sender: [6:7571518966347818071:2120] self: [6:7571518966347818072:0] peer: [5:7571518966044552156:0] socket: 521 qp: -1
2025-11-11T17:20:34.650335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518966347818075:2071] [node 5] ICS10 traffic start
2025-11-11T17:20:34.650365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:34.661713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:34.667975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518966347818035:2078] [node 4] ICP22 created new session: [6:7571518966347818077:2078]
2025-11-11T17:20:34.673903Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518966347818073:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.674041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518962052850687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.674739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518966347818077:2078] [node 4] ICS09 handshake done sender: [6:7571518966347818067:2118] self: [6:7571518966347818068:0] peer: [4:7571518967969048975:0] socket: 510 qp: -1
2025-11-11T17:20:34.681488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518966347818077:2078] [node 4] ICS10 traffic start
2025-11-11T17:20:34.684758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:34.689092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818080:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.716306Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518966347818066:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:34.716541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518962052850688:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.716579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:34.716637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518962052850688:2050] [node 3] ICP22 created new session: [6:7571518966347818081:2050]
2025-11-11T17:20:34.716684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518966347818081:2050] [node 3] ICS09 handshake done sender: [6:7571518966347818066:2117] self: [6:7571518966347818069:0] peer: [3:7571518965974048703:0] socket: 506 qp: -1
2025-11-11T17:20:34.716714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518966347818081:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:34.716748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:34.734140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518966347818035:2078] [node 4] ICP35 event processing took too much time PendingConnection:278(86330576ns) {HandleHandshakeStatus:329(86327445ns) {DropIncomingHandshake:441(719ns) DropOutgoingHandshake:456(6771ns) SwitchToState:178(4377ns) ProcessPendingSessionEvents:495(35310051ns) {ForwardSessionEventToSession:612(35246435ns) ForwardSessionEventToSession:612(5932ns) ForwardSessionEventToSession:612(11339ns) ForwardSessionEventToSession:612(3359ns) ForwardSessionEventToSession:612(4103ns) ForwardSessionEventToSession:612(6093ns) ForwardSessionEventToSession:612(6459ns) ForwardSessionEventToSession:612(3190ns) ForwardSessionEventToSession:612(6566ns) ForwardSessionEventToSession:612(3418ns)}}}
2025-11-11T17:20:34.707027Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518966044552168:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:34.707306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518961749584790:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.707394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:34.707443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518961749584790:2049] [node 2] ICP22 created new session: [5:7571518966044552187:2049]
2025-11-11T17:20:34.707489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518966044552187:2049] [node 2] ICS09 handshake done sender: [5:7571518966044552168:2119] self: [5:7571518966044552183:0] peer: [2:7571518968233729174:0] socket: 485 qp: -1
2025-11-11T17:20:34.707538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518966044552187:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:34.707574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:34.545890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518967969048985:2100] [node 5] ICS10 traffic start
2025-11-11T17:20:34.666945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518965974048729:2107] [node 5] ICS10 traffic start
2025-11-11T17:20:34.507860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:34.527071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518968233729183:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42812
2025-11-11T17:20:34.558619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518968233729160:2133] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(215765497ns) {Configure:104(215760808ns) {ProcessConfigured:135(23984931ns) {StartInitialHandshake:157(23981176ns) {DropHandshakes:469(2804ns) {DropIncomingHandshake:441(512ns) DropOutgoingHandshake:456(345ns)} GenerateSessionVirtualId:988(1061ns) PrepareNewSessionHandshake:476(3473ns) {SwitchToState:178(1536ns)}}}}}
2025-11-11T17:20:34.561745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518968233729169:2138] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:34.562439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518963938761776:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.562526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:34.562576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518963938761776:2048] [node 1] ICP22 created new session: [2:7571518968233729185:2048]
2025-11-11T17:20:34.562626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518968233729185:2048] [node 1] ICS09 handshake done sender: [2:7571518968233729169:2138] self: [2:7571518968233729178:0] peer: [1:7571518965102655940:0] socket: 488 qp: -1
2025-11-11T17:20:34.562664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518968233729185:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:34.562699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:34.615328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518968233729160:2133] [node 6] ICP35 event processing took too much time PendingConnection:278(18146265ns) {IncomingHandshake:261(18141508ns) {DropIncomingHandshake:441(559ns) PrepareNewSessionHandshake:476(1731ns) {SwitchToState:178(744ns)} GenerateSessionVirtualId:988(722ns) IssueIncomingHandshakeReply:193(10568ns)}}
2025-11-11T17:20:34.632130Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729162:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.632165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518968233729162:2134] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.639058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518968233729188:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42812
2025-11-11T17:20:34.642591Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518968233729162:2134] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:34.669922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518963938761777:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.670008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:34.670067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518963938761777:2049] [node 3] ICP22 created new session: [2:7571518968233729191:2049]
2025-11-11T17:20:34.670122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518968233729191:2049] [node 3] ICS09 handshake done sender: [2:7571518968233729162:2134] self: [2:7571518968233729161:0] peer: [3:7571518965974048724:0] socket: 490 qp: -1
2025-11-11T17:20:34.670154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518968233729191:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:34.670190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:34.670649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729188:2143] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.671026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518963938761777:2049] [node 3] ICP18 handshake (actor [2:7571518968233729188:2143]) is too old
2025-11-11T17:20:34.686021Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729175:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.686054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518968233729175:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.697680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518968233729184:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.697741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518968233729184:2142] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.698075Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518968233729175:2140] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:34.737940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518968233729163:2121] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.758290Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518968233729184:2142] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.764156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518966347818083:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37661
2025-11-11T17:20:34.766167Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518966347818073:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:34.766528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518962052850687:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.766595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:34.766648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518962052850687:2049] [node 2] ICP22 created new session: [6:7571518966347818084:2049]
2025-11-11T17:20:34.766693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518966347818084:2049] [node 2] ICS09 handshake done sender: [6:7571518966347818073:2121] self: [6:7571518966347818078:0] peer: [2:7571518968233729177:0] socket: 530 qp: -1
2025-11-11T17:20:34.766738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518966347818084:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:34.766805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:34.545949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:34.570254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518967969048964:2100] [node 5] ICP35 event processing took too much time PendingConnection:278(81305814ns) {HandleHandshakeStatus:329(81301608ns) {DropIncomingHandshake:441(17050ns) DropOutgoingHandshake:456(459ns) SwitchToState:178(843ns) ProcessPendingSessionEvents:495(118358ns) {ForwardSessionEventToSession:612(71258ns) ForwardSessionEventToSession:612(3421ns) ForwardSessionEventToSession:612(10858ns) ForwardSessionEventToSession:612(2716ns) ForwardSessionEventToSession:612(2104ns) ForwardSessionEventToSession:612(7630ns) ForwardSessionEventToSession:612(3196ns) ForwardSessionEventToSession:612(1934ns) ForwardSessionEventToSession:612(5064ns) ForwardSessionEventToSession:612(2559ns)}}}
2025-11-11T17:20:34.571283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518967969048990:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18008
2025-11-11T17:20:34.582045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518963674081606:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(56242477ns) {HandleHandshakeStatus:329(56238319ns) {DropIncomingHandshake:441(715ns) DropOutgoingHandshake:456(5731ns) SwitchToState:178(428ns) ProcessPendingSessionEvents:495(16029406ns) {ForwardSessionEventToSession:612(15965584ns) ForwardSessionEventToSession:612(7867ns) ForwardSessionEventToSession:612(8864ns) ForwardSessionEventToSession:612(6372ns) ForwardSessionEventToSession:612(6214ns) ForwardSessionEventToSession:612(3208ns) ForwardSessionEventToSession:612(3187ns) ForwardSessionEventToSession:612(9026ns) ForwardSessionEventToSession:612(3577ns) ForwardSessionEventToSession:612(2889ns)}}}
2025-11-11T17:20:34.590482Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518967969048976:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.590512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518967969048976:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:34.597851Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518967969048976:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:34.618270Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518967969048990:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.618335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518967969048974:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.618375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:34.620144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518963674081605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.621993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518967969048974:2109] [node 6] ICP22 created new session: [4:7571518967969048992:2109]
2025-11-11T17:20:34.632359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518967969048992:2109] [node 6] ICS09 handshake done sender: [4:7571518967969048976:2113] self: [4:7571518967969048975:0] peer: [6:7571518966347818068:0] socket: 509 qp: -1
2025-11-11T17:20:34.632410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518967969048992:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:34.642151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:34.646681Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7819) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:34.646838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518963674081605:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7819) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518967969048990:2118] held: no
2025-11-11T17:20:34.648540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518967969048995:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18008
2025-11-11T17:20:34.658622Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518967969048990:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:34.660429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518967969048974:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(41844921ns) {HandleHandshakeStatus:329(41842145ns) {DropIncomingHandshake:441(367ns) DropOutgoingHandshake:456(769ns) SwitchToState:178(627ns) ProcessPendingSessionEvents:495(7415216ns) {ForwardSessionEventToSession:612(7355256ns) ForwardSessionEventToSession:612(4931ns) ForwardSessionEventToSession:612(6948ns) ForwardSessionEventToSession:612(2844ns) ForwardSessionEventToSession:612(5275ns) ForwardSessionEventToSession:612(3332ns) ForwardSessionEventToSession:612(2143ns) ForwardSessionEventToSession:612(9453ns) ForwardSessionEventToSession:612(4709ns) ForwardSessionEventToSession:612(3335ns)}}}
2025-11-11T17:20:34.674584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518963674081605:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.681797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:34.681845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518963674081605:2048] [node 1] ICP22 created new session: [4:7571518967969048996:2048]
2025-11-11T17:20:34.694284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518967969048996:2048] [node 1] ICS09 handshake done sender: [4:7571518967969048990:2118] self: [4:7571518967969048993:0] peer: [1:7571518965102655952:0] socket: 494 qp: -1
2025-11-11T17:20:34.701783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518967969048996:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:34.701851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:34.726056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518967969048998:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18008
2025-11-11T17:20:34.749934Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518967969048998:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.750086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518963674081607:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:34.757727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518963674081605:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(69645690ns) {HandleHandshakeStatus:329(69642261ns) {DropIncomingHandshake:441(1020ns) DropOutgoingHandshake:456(348ns) SwitchToState:178(213ns) ProcessPendingSessionEvents:495(842ns)}}
2025-11-11T17:20:34.666985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:34.685416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518965974048732:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4473
2025-11-11T17:20:34.689508Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15527) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:34.697952Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518965974048704:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:34.706070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518961679081339:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15527) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518965974048712:2118] held: no
2025-11-11T17:20:34.706505Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518965974048712:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:34.714052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518961679081339:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.714143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:34.714201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518961679081339:2049] [node 2] ICP22 created new session: [3:7571518965974048734:2049]
2025-11-11T17:20:34.714248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518965974048734:2049] [node 2] ICS09 handshake done sender: [3:7571518965974048712:2118] self: [3:7571518965974048724:0] peer: [2:7571518968233729161:0] socket: 502 qp: -1
2025-11-11T17:20:34.714286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518965974048734:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:34.714342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:34.714724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518965974048699:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.714820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:34.714869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518965974048699:2108] [node 6] ICP22 created new session: [3:7571518965974048736:2108]
2025-11-11T17:20:34.714905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518965974048736:2108] [node 6] ICS09 handshake done sender: [3:7571518965974048704:2113] self: [3:7571518965974048703:0] peer: [6:7571518966347818069:0] socket: 503 qp: -1
2025-11-11T17:20:34.714939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518965974048736:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:34.777906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:34.714989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:34.791495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518968233729163:2121] [node 5] ICP22 created new session: [2:7571518968233729195:2121]
2025-11-11T17:20:34.791572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518968233729195:2121] [node 5] ICS09 handshake done sender: [2:7571518968233729175:2140] self: [2:7571518968233729174:0] peer: [5:7571518966044552183:0] socket: 512 qp: -1
2025-11-11T17:20:34.791613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518968233729195:2121] [node 5] ICS10 traffic start
2025-11-11T17:20:34.791659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:34.792100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518968233729163:2121] [node 5] ICP35 event processing took too much time PendingConnection:278(53959695ns) {HandleHandshakeStatus:329(53956859ns) {DropIncomingHandshake:441(14769ns) DropOutgoingHandshake:456(550ns) SwitchToState:178(471ns) ProcessPendingSessionEvents:495(105148ns) {ForwardSessionEventToSession:612(57039ns) ForwardSessionEventToSession:612(3583ns) ForwardSessionEventToSession:612(2644ns) ForwardSessionEventToSession:612(8170ns) ForwardSessionEventToSession:612(2332ns) ForwardSessionEventToSession:612(5197ns) ForwardSessionEventToSession:612(2983ns) ForwardSessionEventToSession:612(6233ns) ForwardSessionEventToSession:612(5334ns) ForwardSessionEventToSession:612(2508ns)}}}
2025-11-11T17:20:34.792151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518968233729160:2133] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.792251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:34.792291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518968233729160:2133] [node 6] ICP22 created new session: [2:7571518968233729197:2133]
2025-11-11T17:20:34.792353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518968233729197:2133] [node 6] ICS09 handshake done sender: [2:7571518968233729184:2142] self: [2:7571518968233729177:0] peer: [6:7571518966347818078:0] socket: 526 qp: -1
2025-11-11T17:20:34.792385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518968233729197:2133] [node 6] ICS10 traffic start
2025-11-11T17:20:34.792432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:34.777956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518965974048718:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:34.778000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518965974048718:2120] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:34.796178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518967969049000:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18008
2025-11-11T17:20:34.795896Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518965974048718:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:34.796785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518965974048698:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:34.796866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:34.796907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518965974048698:2109] [node 4] ICP22 created new session: [3:7571518965974048739:2109]
2025-11-11T17:20:34.796950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518965974048739:2109] [node 4] ICS09 handshake done sender: [3:7571518965974048718:2120] self: [3:7571518965974048717:0] peer: [4:7571518967969048999:0] socket: 500 qp: -1
2025-11-11T17:20:34.796976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518965974048739:2109] [node 4] ICS10 traffic start
2025-11-11T17:20:34.797027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:34.798326Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518967969048998:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:34.798650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518963674081607:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:34.798749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:34.798795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518963674081607:2050] [node 3] ICP22 created new session: [4:7571518967969049001:2050]
2025-11-11T17:20:34.798833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518967969049001:2050] [node 3] ICS09 handshake done sender: [4:7571518967969048998:2120] self: [4:7571518967969048999:0] peer: [3:7571518965974048717:0] socket: 481 qp: -1
2025-11-11T17:20:34.798866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518967969049001:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:34.798920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:34.846237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:34.846497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518968233729185:2048] [node 1] ICS01 socket: 488 reason# EndOfStream
2025-11-11T17:20:34.846543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518963938761776:2048] [node 1] ICP30 unregister session Session# [2:7571518968233729185:2048] VirtualId# [2:7571518968233729178:0]
2025-11-11T17:20:34.846583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518968233729185:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.846671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:34.846713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:34.848620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:34.848848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518965974048725:2048] [node 1] ICS01 socket: 477 reason# EndOfStream
2025-11-11T17:20:34.848887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518961679081338:2048] [node 1] ICP30 unregister session Session# [3:7571518965974048725:2048] VirtualId# [3:7571518965974048719:0]
2025-11-11T17:20:34.848921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518965974048725:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.848990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:34.849027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:34.860759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:34.861003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518966044552176:2048] [node 1] ICS01 socket: 518 reason# EndOfStream
2025-11-11T17:20:34.861049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518961749584789:2048] [node 1] ICP30 unregister session Session# [5:7571518966044552176:2048] VirtualId# [5:7571518966044552173:0]
2025-11-11T17:20:34.861082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518966044552176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.861153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:34.861198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:34.863866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:34.864104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518967969048996:2048] [node 1] ICS01 socket: 494 reason# EndOfStream
2025-11-11T17:20:34.864145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518963674081605:2048] [node 1] ICP30 unregister session Session# [4:7571518967969048996:2048] VirtualId# [4:7571518967969048993:0]
2025-11-11T17:20:34.864178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518967969048996:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.864258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:34.864305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:34.864400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:34.864585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518966347818063:2048] [node 1] ICS01 socket: 487 reason# EndOfStream
2025-11-11T17:20:34.864612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518962052850686:2048] [node 1] ICP30 unregister session Session# [6:7571518966347818063:2048] VirtualId# [6:7571518966347818059:0]
2025-11-11T17:20:34.864635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518966347818063:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.864685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:34.864721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:34.959420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:34.959672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518965974048734:2049] [node 2] ICS01 socket: 502 reason# EndOfStream
2025-11-11T17:20:34.959721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518961679081339:2049] [node 2] ICP30 unregister session Session# [3:7571518965974048734:2049] VirtualId# [3:7571518965974048724:0]
2025-11-11T17:20:34.959756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518965974048734:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.959833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:34.959871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:34.969780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:34.970031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571518967969048987:2049] [node 2] ICS07 socket disconnect 493 reason# EndOfStream
2025-11-11T17:20:34.970059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518967969048987:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.970144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:34.970179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571518967969048987:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:34.970786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:34.970965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518966044552187:2049] [node 2] ICS01 socket: 485 reason# EndOfStream
2025-11-11T17:20:34.970995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518961749584790:2049] [node 2] ICP30 unregister session Session# [5:7571518966044552187:2049] VirtualId# [5:7571518966044552183:0]
2025-11-11T17:20:34.971019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518966044552187:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.971080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:34.971114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:34.972107Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15527) Couldn't connect to any resolved address
2025-11-11T17:20:34.972368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518963674081606:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15527) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:34.972413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518967969048987:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:34.972444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518963674081606:2049] [node 2] ICP30 unregister session Session# [4:7571518967969048987:2049] VirtualId# [4:7571518967969048983:0]
2025-11-11T17:20:34.972482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:34.972516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518963674081606:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15527) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:34.972530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15527) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:34.969796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:34.970031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518966347818084:2049] [node 2] ICS01 socket: 530 reason# EndOfStream
2025-11-11T17:20:34.970067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518962052850687:2049] [node 2] ICP30 unregister session Session# [6:7571518966347818084:2049] VirtualId# [6:7571518966347818078:0]
2025-11-11T17:20:34.970101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518966347818084:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:34.970160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:34.970210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:34.981696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518963674081606:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:35.097799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:35.098045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518966347818081:2050] [node 3] ICS01 socket: 506 reason# EndOfStream
2025-11-11T17:20:35.098087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518962052850688:2050] [node 3] ICP30 unregister session Session# [6:7571518966347818081:2050] VirtualId# [6:7571518966347818069:0]
2025-11-11T17:20:35.098119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518966347818081:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:35.098218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:35.098266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:35.098097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:35.098330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518966044552184:2050] [node 3] ICS01 socket: 507 reason# EndOfStream
2025-11-11T17:20:35.098370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518961749584791:2050] [node 3] ICP30 unregister session Session# [5:7571518966044552184:2050] VirtualId# [5:7571518966044552172:0]
2025-11-11T17:20:35.098404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518966044552184:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:35.098535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:35.098589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:35.098698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:35.098890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518967969049001:2050] [node 3] ICS01 socket: 481 reason# EndOfStream
2025-11-11T17:20:35.098986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518963674081607:2050] [node 3] ICP30 unregister session Session# [4:7571518967969049001:2050] VirtualId# [4:7571518967969048999:0]
2025-11-11T17:20:35.099011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518967969049001:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:35.099067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:35.099101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:35.181783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:35.182012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518966044552175:2090] [node 4] ICS01 socket: 480 reason# EndOfStream
2025-11-11T17:20:35.182060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518966044552128:2090] [node 4] ICP30 unregister session Session# [5:7571518966044552175:2090] VirtualId# [5:7571518966044552162:0]
2025-11-11T17:20:35.182094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518966044552175:2090] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:35.182171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:35.182217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:35.185811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:35.186238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518966347818077:2078] [node 4] ICS07 socket disconnect 510 reason# EndOfStream
2025-11-11T17:20:35.186303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518966347818077:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:35.186449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:35.186514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518966347818077:2078] [node 4] ICS15 start handshake
2025-11-11T17:20:35.194087Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22598) Couldn't connect to any resolved address
2025-11-11T17:20:35.196536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518966347818035:2078] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22598) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:35.196625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518966347818077:2078] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:35.196666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518966347818035:2078] [node 4] ICP30 unregister session Session# [6:7571518966347818077:2078] VirtualId# [6:7571518966347818068:0]
2025-11-11T17:20:35.196728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:35.196765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518966347818035:2078] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22598) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:35.196802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22598) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:35.210009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518966347818035:2078] [node 4] ICP33 wake up from error state
2025-11-11T17:20:35.301782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:35.302041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518966347818075:2071] [node 5] ICS01 socket: 521 reason# EndOfStream
2025-11-11T17:20:35.302086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518966347818008:2071] [node 5] ICP30 unregister session Session# [6:7571518966347818075:2071] VirtualId# [6:7571518966347818072:0]
2025-11-11T17:20:35.302117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518966347818075:2071] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:35.302208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:35.302265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:35.477985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518970255581071:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:35.478037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518970255581072:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:35.553943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518971710866429:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:35.554000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518971710866430:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:35.625979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518971024660703:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:35.626027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518971024660704:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:35.669924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518973158055766:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:35.669977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518973158055767:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:35.670009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518973158055768:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:35.733853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518971945063400:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:35.733900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518971945063401:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:35.733930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518971945063402:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:35.851487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518971113784730:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:35.851565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518971113784731:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:35.862903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518971113784732:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:36.003487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518976005833744:2065] [node 6] ICP01 ready to work
2025-11-11T17:20:36.100335Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518971945063401:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(66173323ns) {Configure:104(66167623ns) {ProcessConfigured:135(76685ns) {StartInitialHandshake:157(72988ns) {DropHandshakes:469(3647ns) {DropIncomingHandshake:441(792ns) DropOutgoingHandshake:456(995ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:476(1657ns) {SwitchToState:178(260ns)}}}}}
2025-11-11T17:20:36.073860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518974550548424:2069] [node 5] ICP01 ready to work
2025-11-11T17:20:36.146002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518976005833775:2058] [node 5] ICP01 ready to work
2025-11-11T17:20:36.158268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518977453023114:2071] [node 6] ICP01 ready to work
2025-11-11T17:20:36.142217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518974550548428:2072] [node 6] ICP01 ready to work
2025-11-11T17:20:36.141820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518975319628028:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.141974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518976240030723:2066] [node 4] ICP01 ready to work
2025-11-11T17:20:36.157940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518976005833759:2073] [node 4] ICP01 ready to work
2025-11-11T17:20:36.202493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518975319628037:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.246301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518975408752080:2065] [node 4] ICP01 ready to work
2025-11-11T17:20:36.185758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518977453023113:2069] [node 5] ICP01 ready to work
2025-11-11T17:20:36.177701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518974550548430:2070] [node 4] ICP01 ready to work
2025-11-11T17:20:36.215932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518976005833776:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40460
2025-11-11T17:20:36.185771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518976240030754:2065] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.262385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023133:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.268329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518970255581072:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(146365769ns) {Configure:104(146359028ns) {ProcessConfigured:135(76654ns) {StartInitialHandshake:157(73387ns) {DropHandshakes:469(3223ns) {DropIncomingHandshake:441(769ns) DropOutgoingHandshake:456(601ns)} GenerateSessionVirtualId:988(4769ns) PrepareNewSessionHandshake:476(2930ns) {SwitchToState:178(1055ns)}}}}}
2025-11-11T17:20:36.290780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030765:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.275274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518971113784730:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(153294942ns) {Configure:104(153289583ns) {ProcessConfigured:135(71696ns) {StartInitialHandshake:157(68876ns) {DropHandshakes:469(3241ns) {DropIncomingHandshake:441(781ns) DropOutgoingHandshake:456(512ns)} GenerateSessionVirtualId:988(1029ns) PrepareNewSessionHandshake:476(1792ns) {SwitchToState:178(702ns)}}}}}
2025-11-11T17:20:36.272900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518975319628056:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.227599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518976005833777:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40460
2025-11-11T17:20:36.247091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833777:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.281439Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833776:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.286844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518971710866430:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(320839999ns) {Configure:104(303757724ns) {ProcessConfigured:135(65840ns) {StartInitialHandshake:157(60451ns) {DropHandshakes:469(3394ns) {DropIncomingHandshake:441(843ns) DropOutgoingHandshake:456(491ns)} GenerateSessionVirtualId:988(1186ns) PrepareNewSessionHandshake:476(2151ns) {SwitchToState:178(827ns)}}}}}
2025-11-11T17:20:36.308514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518976005833775:2058] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(161909662ns) {Configure:104(161904675ns) {ProcessConfigured:135(8843455ns) {StartInitialHandshake:157(8840235ns) {DropHandshakes:469(3384ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(656ns)} GenerateSessionVirtualId:988(1174ns) PrepareNewSessionHandshake:476(5264ns) {SwitchToState:178(2710ns)}}}}}
2025-11-11T17:20:36.273845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518974550548428:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(61258150ns) {Configure:104(61252800ns) {ProcessConfigured:135(47417576ns) {StartInitialHandshake:157(47413812ns) {DropHandshakes:469(3163ns) {DropIncomingHandshake:441(781ns) DropOutgoingHandshake:456(656ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:476(19914323ns) {SwitchToState:178(8081ns)}}}}}
2025-11-11T17:20:36.277298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752081:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.319314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518975319628028:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(162791944ns) {Configure:104(162786498ns) {ProcessConfigured:135(81951ns) {StartInitialHandshake:157(78694ns) {DropHandshakes:469(3748ns) {DropIncomingHandshake:441(644ns) DropOutgoingHandshake:456(1187ns)} GenerateSessionVirtualId:988(1217ns) PrepareNewSessionHandshake:476(3058ns) {SwitchToState:178(1057ns)}}}}}
2025-11-11T17:20:36.296442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518973158055766:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(249998323ns) {Configure:104(249987431ns) {ProcessConfigured:135(108618ns) {StartInitialHandshake:157(106037ns) {DropHandshakes:469(3773ns) {DropIncomingHandshake:441(990ns) DropOutgoingHandshake:456(716ns)} GenerateSessionVirtualId:988(934ns) PrepareNewSessionHandshake:476(2519ns) {SwitchToState:178(922ns)}}}}}
2025-11-11T17:20:36.314514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518974550548445:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52212
2025-11-11T17:20:36.336310Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833788:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.336370Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518976005833788:2104] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.339240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518976005833808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40460
2025-11-11T17:20:36.327282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518976240030754:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(62740707ns) {Configure:104(62734864ns) {ProcessConfigured:135(89438ns) {StartInitialHandshake:157(86144ns) {DropHandshakes:469(4168ns) {DropIncomingHandshake:441(725ns) DropOutgoingHandshake:456(1092ns)} GenerateSessionVirtualId:988(829ns) PrepareNewSessionHandshake:476(2400ns) {SwitchToState:178(1053ns)}}}}}
2025-11-11T17:20:36.307142Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518977453023133:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.277490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752082:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.334633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518971945063400:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(327584634ns) {Configure:104(327576685ns) {ProcessConfigured:135(111406ns) {StartInitialHandshake:157(107807ns) {DropHandshakes:469(4254ns) {DropIncomingHandshake:441(1006ns) DropOutgoingHandshake:456(878ns)} GenerateSessionVirtualId:988(1014ns) PrepareNewSessionHandshake:476(2508ns) {SwitchToState:178(739ns)}}}}}
2025-11-11T17:20:36.283135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518975408752096:2062] [node 5] ICP01 ready to work
2025-11-11T17:20:36.313987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518973158055767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.344419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518974550548431:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52212
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.353345Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518976005833788:2104] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:36.370361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518975319628037:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(50762936ns) {Configure:104(50756202ns) {ProcessConfigured:135(12759549ns) {StartInitialHandshake:157(12754998ns) {DropHandshakes:469(2782ns) {DropIncomingHandshake:441(685ns) DropOutgoingHandshake:456(410ns)} GenerateSessionVirtualId:988(885ns) PrepareNewSessionHandshake:476(4818ns) {SwitchToState:178(1991ns)}}}}}
2025-11-11T17:20:36.365676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833746:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.291117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518971113784732:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(359028105ns) {Configure:104(359021479ns) {ProcessConfigured:135(10574824ns) {StartInitialHandshake:157(10571290ns) {DropHandshakes:469(3375ns) {DropIncomingHandshake:441(927ns) DropOutgoingHandshake:456(602ns)} GenerateSessionVirtualId:988(819ns) PrepareNewSessionHandshake:476(3428ns) {SwitchToState:178(1440ns)}}}}}
2025-11-11T17:20:36.371209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518975319628064:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58159
2025-11-11T17:20:36.411806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023140:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.410998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518976240030723:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(166978439ns) {Configure:104(146187930ns) {ProcessConfigured:135(83633ns) {StartInitialHandshake:157(80891ns) {DropHandshakes:469(3586ns) {DropIncomingHandshake:441(789ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(707ns) PrepareNewSessionHandshake:476(2100ns) {SwitchToState:178(711ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.405734Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548431:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.417709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518976240030765:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.291337Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518975408752081:2102] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.365748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518976005833746:2066] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.437786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518971945063400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.371367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518975319628067:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58159
2025-11-11T17:20:36.292091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752099:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.310200Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518975408752082:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.461808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023141:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.432723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518970255581071:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(422309136ns) {Configure:104(422299633ns) {ProcessConfigured:135(15856241ns) {StartInitialHandshake:157(6797526ns) {DropHandshakes:469(4530ns) {DropIncomingHandshake:441(2059ns) DropOutgoingHandshake:456(501ns)} GenerateSessionVirtualId:988(6400ns) PrepareNewSessionHandshake:476(4635ns) {SwitchToState:178(1982ns)}}}}}
2025-11-11T17:20:36.322834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518971113784731:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.330461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752104:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.386457Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518975408752099:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.447521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518971113784730:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.448125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752105:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.458326Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518975408752104:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.486273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030777:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.438302Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518976005833746:2066] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.371465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518975319628068:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58159
2025-11-11T17:20:36.432949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548439:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.438552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518976005833811:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40460
2025-11-11T17:20:36.494290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030775:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.371576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518975319628069:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58159
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.497649Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.432972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518974550548439:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.373478Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628064:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.439235Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833811:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.439277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518976005833759:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.439353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:36.439405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518976005833759:2073] [node 4] ICP22 created new session: [2:7571518976005833812:2073]
2025-11-11T17:20:36.439450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518976005833812:2073] [node 4] ICS09 handshake done sender: [2:7571518976005833788:2104] self: [2:7571518976005833787:0] peer: [4:7571518977453023139:0] socket: 484 qp: -1
2025-11-11T17:20:36.439484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518976005833812:2073] [node 4] ICS10 traffic start
2025-11-11T17:20:36.439551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:36.440702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518976005833744:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.440738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:36.440773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518976005833744:2065] [node 6] ICP22 created new session: [2:7571518976005833814:2065]
2025-11-11T17:20:36.440823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518976005833814:2065] [node 6] ICS09 handshake done sender: [2:7571518976005833746:2066] self: [2:7571518976005833745:0] peer: [6:7571518975408752103:0] socket: 483 qp: -1
2025-11-11T17:20:36.440853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518976005833814:2065] [node 6] ICS10 traffic start
2025-11-11T17:20:36.440912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:36.522598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518976005833819:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40460
2025-11-11T17:20:36.523393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833819:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.523554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518971710866429:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.534506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518976005833821:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40460
2025-11-11T17:20:36.555736Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518976005833819:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:36.505912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030778:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.530010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518975408752080:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(271611314ns) {Configure:104(271603594ns) {ProcessConfigured:135(18409850ns) {StartInitialHandshake:157(18405650ns) {DropHandshakes:469(2386ns) {DropIncomingHandshake:441(589ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(921ns) PrepareNewSessionHandshake:476(3774ns) {SwitchToState:178(1692ns)}}}}}
2025-11-11T17:20:36.441983Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518974550548439:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:36.373575Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628068:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.394200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518971024660704:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(382750047ns) {Configure:104(382745977ns) {ProcessConfigured:135(73656ns) {StartInitialHandshake:157(71072ns) {DropHandshakes:469(3880ns) {DropIncomingHandshake:441(877ns) DropOutgoingHandshake:456(717ns)} GenerateSessionVirtualId:988(1249ns) PrepareNewSessionHandshake:476(2537ns) {SwitchToState:178(981ns)}}}}}
2025-11-11T17:20:36.571277Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:36.577840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518971710866429:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518976005833819:2119] held: no
2025-11-11T17:20:36.581673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518971710866429:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.509968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518973158055767:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518977453023133:2110] held: no
2025-11-11T17:20:36.546195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518975408752096:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(224810096ns) {Configure:104(224805257ns) {ProcessConfigured:135(85056ns) {StartInitialHandshake:157(82525ns) {DropHandshakes:469(2382ns) {DropIncomingHandshake:441(526ns) DropOutgoingHandshake:456(554ns)} GenerateSessionVirtualId:988(821ns) PrepareNewSessionHandshake:476(1421ns) {SwitchToState:178(409ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.448344Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548433:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.448373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518974550548433:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.500391Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548437:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.500424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518974550548437:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.501625Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518974550548433:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:36.502146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518974550548424:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.502197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:36.502249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518974550548424:2069] [node 5] ICP22 created new session: [1:7571518974550548451:2069]
2025-11-11T17:20:36.502305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518974550548451:2069] [node 5] ICS09 handshake done sender: [1:7571518974550548433:2110] self: [1:7571518974550548432:0] peer: [5:7571518976240030776:0] socket: 488 qp: -1
2025-11-11T17:20:36.502348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518974550548451:2069] [node 5] ICS10 traffic start
2025-11-11T17:20:36.502392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:36.502882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518970255581072:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.502913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:36.502958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518970255581072:2049] [node 3] ICP22 created new session: [1:7571518974550548453:2049]
2025-11-11T17:20:36.503005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518974550548453:2049] [node 3] ICS09 handshake done sender: [1:7571518974550548439:2113] self: [1:7571518974550548438:0] peer: [3:7571518975319628083:0] socket: 492 qp: -1
2025-11-11T17:20:36.503035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518974550548453:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:36.503059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:36.515223Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518974550548437:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:36.515587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518974550548428:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.515668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.510210Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518977453023141:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.515721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518974550548428:2072] [node 6] ICP22 created new session: [1:7571518974550548458:2072]
2025-11-11T17:20:36.515792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518974550548458:2072] [node 6] ICS09 handshake done sender: [1:7571518974550548437:2112] self: [1:7571518974550548435:0] peer: [6:7571518975408752106:0] socket: 491 qp: -1
2025-11-11T17:20:36.594147Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518976240030778:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.546638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518975408752080:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.589692Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833808:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.395610Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628067:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.431092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518971024660703:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(147899784ns) {Configure:104(147892879ns) {ProcessConfigured:135(100607ns) {StartInitialHandshake:157(97732ns) {DropHandshakes:469(3028ns) {DropIncomingHandshake:441(395ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(582ns) PrepareNewSessionHandshake:476(1264ns) {SwitchToState:178(519ns)}}}}}
2025-11-11T17:20:36.605877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518976240030775:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.510294Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518977453023133:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:36.589958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.515833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518974550548458:2072] [node 6] ICS10 traffic start
2025-11-11T17:20:36.546993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518975408752096:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.521744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518977453023113:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(198461002ns) {Configure:104(198452714ns) {ProcessConfigured:135(19918226ns) {StartInitialHandshake:157(19914248ns) {DropHandshakes:469(1572ns) {DropIncomingHandshake:441(294ns) DropOutgoingHandshake:456(243ns)} GenerateSessionVirtualId:988(338ns) PrepareNewSessionHandshake:476(4020ns) {SwitchToState:178(626ns)}}}}}
2025-11-11T17:20:36.431311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518971024660703:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.614869Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518976240030769:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.515866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:36.590018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518971710866429:2048] [node 1] ICP22 created new session: [2:7571518976005833822:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.547171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518975408752082:2103] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.522007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518973158055768:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.590062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518976005833822:2048] [node 1] ICS09 handshake done sender: [2:7571518976005833819:2119] self: [2:7571518976005833820:0] peer: [1:7571518974550548446:0] socket: 524 qp: -1
2025-11-11T17:20:36.614918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518976240030769:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.431887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518975319628056:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(111426744ns) {Configure:104(111422281ns) {ProcessConfigured:135(17258089ns) {StartInitialHandshake:157(17256218ns) {DropHandshakes:469(3473ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(597ns)} GenerateSessionVirtualId:988(559ns) PrepareNewSessionHandshake:476(1735ns) {SwitchToState:178(137ns)}}}}}
2025-11-11T17:20:36.523817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548447:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.615036Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518976240030765:2110] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:36.635054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518971945063401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.635130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518971945063402:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.635173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518971945063400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.635218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:36.635261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518971945063400:2048] [node 1] ICP22 created new session: [5:7571518976240030782:2048]
2025-11-11T17:20:36.635314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518976240030782:2048] [node 1] ICS09 handshake done sender: [5:7571518976240030765:2110] self: [5:7571518976240030776:0] peer: [1:7571518974550548432:0] socket: 490 qp: -1
2025-11-11T17:20:36.635352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518976240030782:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:36.635393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:36.642495Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518976240030769:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:36.643690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518976240030754:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.643800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:36.643852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518976240030754:2065] [node 6] ICP22 created new session: [5:7571518976240030786:2065]
2025-11-11T17:20:36.643902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518976240030786:2065] [node 6] ICS09 handshake done sender: [5:7571518976240030769:2113] self: [5:7571518976240030768:0] peer: [6:7571518975408752113:0] socket: 500 qp: -1
2025-11-11T17:20:36.643937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518976240030786:2065] [node 6] ICS10 traffic start
2025-11-11T17:20:36.643974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:36.682159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030790:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.686077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030779:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.705656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518976240030779:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.547894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518971113784731:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.523842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518974550548447:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.538049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518973158055767:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.590104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518976005833822:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:36.450299Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628069:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.547944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:36.538109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:36.590143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:36.474224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518975319628086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58159
2025-11-11T17:20:36.720025Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:36.734163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518971945063401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518976240030778:2118] held: no
2025-11-11T17:20:36.734352Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518976240030778:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:36.742129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.742307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571518976240030754:2065] [node 6] ICP18 handshake (actor [5:7571518976240030779:2119]) is too old
2025-11-11T17:20:36.742369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518971945063401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.742447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:36.742498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518971945063401:2049] [node 2] ICP22 created new session: [5:7571518976240030791:2049]
2025-11-11T17:20:36.742544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518976240030791:2049] [node 2] ICS09 handshake done sender: [5:7571518976240030778:2118] self: [5:7571518976240030780:0] peer: [2:7571518976005833805:0] socket: 520 qp: -1
2025-11-11T17:20:36.742583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518976240030791:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:36.742620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:36.743563Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518976240030788:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.743691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518976240030723:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.745500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030794:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.525371Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518974550548447:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:36.545686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518973158055767:2049] [node 2] ICP22 created new session: [4:7571518977453023146:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.547986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518971113784731:2049] [node 2] ICP22 created new session: [6:7571518975408752114:2049]
2025-11-11T17:20:36.601823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518976005833744:2065] [node 6] ICP18 handshake (actor [2:7571518976005833808:2117]) is too old
2025-11-11T17:20:36.500401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518975319628087:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58159
2025-11-11T17:20:36.525663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518970255581071:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:36.573636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518977453023146:2049] [node 2] ICS09 handshake done sender: [4:7571518977453023133:2110] self: [4:7571518977453023139:0] peer: [2:7571518976005833787:0] socket: 485 qp: -1
2025-11-11T17:20:36.794061Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12259) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:36.654007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833806:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.548033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518975408752114:2049] [node 2] ICS09 handshake done sender: [6:7571518975408752082:2103] self: [6:7571518975408752103:0] peer: [2:7571518976005833745:0] socket: 487 qp: -1
2025-11-11T17:20:36.654058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518976005833806:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.548066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518975408752114:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:36.548098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:36.548669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752111:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.549287Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518975408752099:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:36.549490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518971113784730:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.549538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:36.549574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518971113784730:2048] [node 1] ICP22 created new session: [6:7571518975408752116:2048]
2025-11-11T17:20:36.554001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518975408752116:2048] [node 1] ICS09 handshake done sender: [6:7571518975408752099:2114] self: [6:7571518975408752106:0] peer: [1:7571518974550548435:0] socket: 493 qp: -1
2025-11-11T17:20:36.554049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518975408752116:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:36.554095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:36.600689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752119:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.637688Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518975408752081:2102] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:36.643174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752121:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.643303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752122:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.653070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518975408752080:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.653165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:36.653218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518975408752080:2065] [node 4] ICP22 created new session: [6:7571518975408752123:2065]
2025-11-11T17:20:36.653267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518975408752123:2065] [node 4] ICS09 handshake done sender: [6:7571518975408752081:2102] self: [6:7571518975408752112:0] peer: [4:7571518977453023115:0] socket: 486 qp: -1
2025-11-11T17:20:36.653312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518975408752123:2065] [node 4] ICS10 traffic start
2025-11-11T17:20:36.653387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:36.688008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518975408752121:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.688162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518971113784732:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.698061Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518975408752104:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:36.720246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518975408752096:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.741707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:36.754044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518975408752096:2062] [node 5] ICP22 created new session: [6:7571518975408752127:2062]
2025-11-11T17:20:36.757752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518975408752080:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(16841417ns) {HandleHandshakeStatus:329(16838180ns) {DropIncomingHandshake:441(955ns) DropOutgoingHandshake:456(8541ns) SwitchToState:178(834ns) ProcessPendingSessionEvents:495(16296052ns) {ForwardSessionEventToSession:612(69860ns) ForwardSessionEventToSession:612(4476ns) ForwardSessionEventToSession:612(2717ns) ForwardSessionEventToSession:612(7428ns) ForwardSessionEventToSession:612(5922ns) ForwardSessionEventToSession:612(2913ns) ForwardSessionEventToSession:612(7426ns) ForwardSessionEventToSession:612(3288ns) ForwardSessionEventToSession:612(16173852ns) ForwardSessionEventToSession:612(6590ns)}}}
2025-11-11T17:20:36.774184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518975408752127:2062] [node 5] ICS09 handshake done sender: [6:7571518975408752104:2116] self: [6:7571518975408752113:0] peer: [5:7571518976240030768:0] socket: 502 qp: -1
2025-11-11T17:20:36.781723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518975408752127:2062] [node 5] ICS10 traffic start
2025-11-11T17:20:36.793753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:36.830450Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12259) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:36.574854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628066:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.846677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518975408752096:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(126167159ns) {HandleHandshakeStatus:329(126163733ns) {DropIncomingHandshake:441(943ns) DropOutgoingHandshake:456(7274ns) SwitchToState:178(4007ns) ProcessPendingSessionEvents:495(12651013ns) {ForwardSessionEventToSession:612(12594162ns) ForwardSessionEventToSession:612(7219ns) ForwardSessionEventToSession:612(6850ns) ForwardSessionEventToSession:612(4095ns) ForwardSessionEventToSession:612(3839ns) ForwardSessionEventToSession:612(5530ns) ForwardSessionEventToSession:612(3681ns) ForwardSessionEventToSession:612(5829ns) ForwardSessionEventToSession:612(5545ns) ForwardSessionEventToSession:612(2898ns)}}}
2025-11-11T17:20:36.540643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518974550548461:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52212
2025-11-11T17:20:36.809924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518976240030789:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62546
2025-11-11T17:20:36.853855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518975408752126:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40273
2025-11-11T17:20:36.577679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518977453023146:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:36.654160Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518976005833803:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.654189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518976005833803:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.854580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518971113784732:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12259) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518975408752121:2122] held: no
2025-11-11T17:20:36.541475Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548461:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.546780Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548445:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.558054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:36.566725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518974550548428:2072] [node 6] ICP18 handshake (actor [1:7571518974550548445:2115]) is too old
2025-11-11T17:20:36.571123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518970255581071:2048] [node 2] ICP22 created new session: [1:7571518974550548463:2048]
2025-11-11T17:20:36.578770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518974550548463:2048] [node 2] ICS09 handshake done sender: [1:7571518974550548447:2116] self: [1:7571518974550548446:0] peer: [2:7571518976005833820:0] socket: 523 qp: -1
2025-11-11T17:20:36.578819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518974550548463:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:36.578869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:36.677778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518974550548441:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.677807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518974550548441:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.693721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518970255581071:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(101085456ns) {HandleHandshakeStatus:329(101082168ns) {DropIncomingHandshake:441(13855ns) DropOutgoingHandshake:456(830ns) SwitchToState:178(4350ns) ProcessPendingSessionEvents:495(47750868ns) {ForwardSessionEventToSession:612(22744769ns) ForwardSessionEventToSession:612(16404ns) ForwardSessionEventToSession:612(11912114ns) ForwardSessionEventToSession:612(6083ns) ForwardSessionEventToSession:612(3829ns) ForwardSessionEventToSession:612(2282236ns) ForwardSessionEventToSession:612(5788ns) ForwardSessionEventToSession:612(10165438ns) ForwardSessionEventToSession:612(579111ns) ForwardSessionEventToSession:612(3682ns)}}}
2025-11-11T17:20:36.698239Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518974550548441:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:36.746011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518974550548430:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.746103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:36.746157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518974550548430:2070] [node 4] ICP22 created new session: [1:7571518974550548466:2070]
2025-11-11T17:20:36.746211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518974550548466:2070] [node 4] ICS09 handshake done sender: [1:7571518974550548441:2114] self: [1:7571518974550548440:0] peer: [4:7571518977453023158:0] socket: 522 qp: -1
2025-11-11T17:20:36.746246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518974550548466:2070] [node 4] ICS10 traffic start
2025-11-11T17:20:36.746286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:36.806276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518974550548430:2070] [node 4] ICP35 event processing took too much time PendingConnection:278(59947429ns) {HandleHandshakeStatus:329(59945016ns) {DropIncomingHandshake:441(13520ns) DropOutgoingHandshake:456(589ns) SwitchToState:178(611ns) ProcessPendingSessionEvents:495(24053603ns) {ForwardSessionEventToSession:612(23993520ns) ForwardSessionEventToSession:612(8037ns) ForwardSessionEventToSession:612(8022ns) ForwardSessionEventToSession:612(5412ns) ForwardSessionEventToSession:612(2776ns) ForwardSessionEventToSession:612(8707ns) ForwardSessionEventToSession:612(3167ns) ForwardSessionEventToSession:612(3286ns) ForwardSessionEventToSession:612(5585ns) ForwardSessionEventToSession:612(2943ns)}}}
2025-11-11T17:20:36.574881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518975319628066:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.669529Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518976005833806:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:36.577729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:36.586612Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518977453023116:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.586649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518977453023116:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.600905Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518977453023116:2094] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:36.601201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518977453023114:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.601242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:36.601291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518977453023114:2071] [node 6] ICP22 created new session: [4:7571518977453023150:2071]
2025-11-11T17:20:36.601337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518977453023150:2071] [node 6] ICS09 handshake done sender: [4:7571518977453023116:2094] self: [4:7571518977453023115:0] peer: [6:7571518975408752112:0] socket: 482 qp: -1
2025-11-11T17:20:36.601375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518977453023150:2071] [node 6] ICS10 traffic start
2025-11-11T17:20:36.601408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:36.601800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518973158055767:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(59807778ns) {HandleHandshakeStatus:329(59803709ns) {DropIncomingHandshake:441(1021ns) DropOutgoingHandshake:456(678ns) SwitchToState:178(717ns) ProcessPendingSessionEvents:495(1272ns)}}
2025-11-11T17:20:36.618326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023152:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.621645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023145:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.642058Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518977453023141:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:36.646104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518973158055768:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.646190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:36.646237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518973158055768:2050] [node 3] ICP22 created new session: [4:7571518977453023156:2050]
2025-11-11T17:20:36.646288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518977453023156:2050] [node 3] ICS09 handshake done sender: [4:7571518977453023141:2116] self: [4:7571518977453023144:0] peer: [3:7571518975319628065:0] socket: 509 qp: -1
2025-11-11T17:20:36.646323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518977453023156:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:36.646383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:36.649794Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518977453023145:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.666670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518973158055766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.675971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023154:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.682577Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:20:36.682822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023155:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.694634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518977453023159:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60143
2025-11-11T17:20:36.722167Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518977453023145:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:36.722517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518973158055766:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.722590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:36.722637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518973158055766:2048] [node 1] ICP22 created new session: [4:7571518977453023160:2048]
2025-11-11T17:20:36.722702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518977453023160:2048] [node 1] ICS09 handshake done sender: [4:7571518977453023145:2118] self: [4:7571518977453023158:0] peer: [1:7571518974550548440:0] socket: 527 qp: -1
2025-11-11T17:20:36.722736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518977453023160:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:36.722774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:36.743949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518977453023143:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.743973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518977453023143:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.749805Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518977453023155:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.758151Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518977453023143:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:36.813824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518977453023113:2069] [node 5] ICP35 event processing took too much time PendingConnection:278(19984184ns) {IncomingHandshake:261(19981001ns) {DropIncomingHandshake:441(1018ns) PrepareNewSessionHandshake:476(2435ns) {SwitchToState:178(1044ns)} GenerateSessionVirtualId:988(870ns) IssueIncomingHandshakeReply:193(11069ns)}}
2025-11-11T17:20:36.819396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518977453023113:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.829793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:36.829864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518977453023113:2069] [node 5] ICP22 created new session: [4:7571518977453023164:2069]
2025-11-11T17:20:36.829917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518977453023164:2069] [node 5] ICS09 handshake done sender: [4:7571518977453023143:2117] self: [4:7571518977453023142:0] peer: [5:7571518976240030793:0] socket: 503 qp: -1
2025-11-11T17:20:36.829974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518977453023164:2069] [node 5] ICS10 traffic start
2025-11-11T17:20:36.830935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:36.866965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518977453023113:2069] [node 5] ICP35 event processing took too much time PendingConnection:278(52744604ns) {HandleHandshakeStatus:329(47262762ns) {DropIncomingHandshake:441(26224ns) DropOutgoingHandshake:456(325ns) SwitchToState:178(4609ns) ProcessPendingSessionEvents:495(8876418ns) {ForwardSessionEventToSession:612(8814473ns) ForwardSessionEventToSession:612(7216ns) ForwardSessionEventToSession:612(8052ns) ForwardSessionEventToSession:612(3041ns) ForwardSessionEventToSession:612(3070ns) ForwardSessionEventToSession:612(5632ns) ForwardSessionEventToSession:612(10385ns) ForwardSessionEventToSession:612(2821ns) ForwardSessionEventToSession:612(5144ns) ForwardSessionEventToSession:612(2759ns)}}}
2025-11-11T17:20:36.810202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518971945063402:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12259) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518976240030775:2116] held: no
2025-11-11T17:20:36.810462Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518976240030788:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:36.818172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518976240030723:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.818288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:36.818349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518976240030723:2066] [node 4] ICP22 created new session: [5:7571518976240030795:2066]
2025-11-11T17:20:36.818398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518976240030795:2066] [node 4] ICS09 handshake done sender: [5:7571518976240030788:2120] self: [5:7571518976240030793:0] peer: [4:7571518977453023142:0] socket: 536 qp: -1
2025-11-11T17:20:36.818439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518976240030795:2066] [node 4] ICS10 traffic start
2025-11-11T17:20:36.818488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:36.819005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518976240030775:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:36.830895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518971945063402:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.869661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:36.877792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518971945063402:2050] [node 3] ICP22 created new session: [5:7571518976240030797:2050]
2025-11-11T17:20:36.870100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518971113784732:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(28282304ns) {HandleHandshakeStatus:395(24507607ns) {LogHandshakeFail:472(4975717ns) {UpdateErrorStateLog:544(6144ns)} DropOutgoingHandshake:456(2742ns) ProcessPendingSessionEvents:495(7619704ns) {DropSessionEvent:512(17485ns) DropSessionEvent:512(7067498ns) DropSessionEvent:512(5717ns) DropSessionEvent:512(6721ns) DropSessionEvent:512(7766ns) DropSessionEvent:512(6207ns) DropSessionEvent:512(19609ns) DropSessionEvent:512(6309ns) DropSessionEvent:512(6624ns)}}}
2025-11-11T17:20:36.870247Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518975408752121:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:36.575472Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518975319628068:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:36.593699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518971024660703:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.593776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:36.593888Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518975319628066:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:36.597725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628087:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.614281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518971024660703:2048] [node 1] ICP22 created new session: [3:7571518975319628089:2048]
2025-11-11T17:20:36.621986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518971024660704:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:36.622340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518975319628089:2048] [node 1] ICS09 handshake done sender: [3:7571518975319628068:2112] self: [3:7571518975319628083:0] peer: [1:7571518974550548438:0] socket: 498 qp: -1
2025-11-11T17:20:36.633696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518975319628028:2059] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.633806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:36.637669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518975319628089:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:36.637718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:36.637916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518975319628028:2059] [node 4] ICP22 created new session: [3:7571518975319628092:2059]
2025-11-11T17:20:36.637972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518975319628092:2059] [node 4] ICS09 handshake done sender: [3:7571518975319628066:2110] self: [3:7571518975319628065:0] peer: [4:7571518977453023144:0] socket: 506 qp: -1
2025-11-11T17:20:36.638011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518975319628092:2059] [node 4] ICS10 traffic start
2025-11-11T17:20:36.638079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:36.638687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628072:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.638713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518975319628072:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.686642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518971024660703:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(107251795ns) {HandleHandshakeStatus:329(107249111ns) {DropIncomingHandshake:441(2660ns) DropOutgoingHandshake:456(10639ns) SwitchToState:178(622ns) ProcessPendingSessionEvents:495(9002441ns) {ForwardSessionEventToSession:612(8940580ns) ForwardSessionEventToSession:612(9349ns) ForwardSessionEventToSession:612(2765ns) ForwardSessionEventToSession:612(3103ns) ForwardSessionEventToSession:612(8394ns) ForwardSessionEventToSession:612(3102ns) ForwardSessionEventToSession:612(8217ns) ForwardSessionEventToSession:612(3100ns) ForwardSessionEventToSession:612(5146ns) ForwardSessionEventToSession:612(5681ns)}}}
2025-11-11T17:20:36.687622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518975319628096:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58159
2025-11-11T17:20:36.688854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518975319628084:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:36.688881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518975319628084:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:36.697748Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518975319628087:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:36.705686Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518975319628072:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:36.722091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518971024660704:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.722165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:36.722216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518971024660704:2049] [node 2] ICP22 created new session: [3:7571518975319628098:2049]
2025-11-11T17:20:36.722271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518975319628098:2049] [node 2] ICS09 handshake done sender: [3:7571518975319628087:2120] self: [3:7571518975319628090:0] peer: [2:7571518976005833802:0] socket: 519 qp: -1
2025-11-11T17:20:36.722315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518975319628098:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:36.722382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:36.738745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518975319628084:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:36.767842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518975319628037:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.767949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:36.768008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518975319628037:2069] [node 5] ICP22 created new session: [3:7571518975319628100:2069]
2025-11-11T17:20:36.768069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518975319628100:2069] [node 5] ICS09 handshake done sender: [3:7571518975319628072:2115] self: [3:7571518975319628071:0] peer: [5:7571518976240030781:0] socket: 507 qp: -1
2025-11-11T17:20:36.768114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518975319628100:2069] [node 5] ICS10 traffic start
2025-11-11T17:20:36.768174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:36.768919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518975319628056:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.769023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:36.769060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518975319628056:2063] [node 6] ICP22 created new session: [3:7571518975319628102:2063]
2025-11-11T17:20:36.769110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518975319628102:2063] [node 6] ICS09 handshake done sender: [3:7571518975319628084:2118] self: [3:7571518975319628080:0] peer: [6:7571518975408752125:0] socket: 531 qp: -1
2025-11-11T17:20:36.769140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518975319628102:2063] [node 6] ICS10 traffic start
2025-11-11T17:20:36.769197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:36.822218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518971024660704:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(99914356ns) {HandleHandshakeStatus:329(99909572ns) {DropIncomingHandshake:441(1595ns) DropOutgoingHandshake:456(8542ns) SwitchToState:178(757ns) ProcessPendingSessionEvents:495(31517739ns) {ForwardSessionEventToSession:612(11383059ns) ForwardSessionEventToSession:612(8167ns) ForwardSessionEventToSession:612(8852ns) ForwardSessionEventToSession:612(2796ns) ForwardSessionEventToSession:612(7914778ns) ForwardSessionEventToSession:612(4045604ns) ForwardSessionEventToSession:612(4124ns) ForwardSessionEventToSession:612(8114919ns) ForwardSessionEventToSession:612(5208ns) ForwardSessionEventToSession:612(2866ns)}}}
2025-11-11T17:20:36.884134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518976240030797:2050] [node 3] ICS09 handshake done sender: [5:7571518976240030775:2116] self: [5:7571518976240030781:0] peer: [3:7571518975319628071:0] socket: 508 qp: -1
2025-11-11T17:20:36.884209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518976240030797:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:36.884293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:36.884563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518971945063402:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(53548105ns) {HandleHandshakeStatus:329(53544798ns) {DropIncomingHandshake:441(903ns) DropOutgoingHandshake:456(695ns) SwitchToState:178(576ns) ProcessPendingSessionEvents:495(908ns)}}
2025-11-11T17:20:36.677755Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518976005833803:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:36.677992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518976005833775:2058] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.678083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:36.678138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518976005833775:2058] [node 5] ICP22 created new session: [2:7571518976005833826:2058]
2025-11-11T17:20:36.678188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518976005833826:2058] [node 5] ICS09 handshake done sender: [2:7571518976005833806:2116] self: [2:7571518976005833805:0] peer: [5:7571518976240030780:0] socket: 518 qp: -1
2025-11-11T17:20:36.678228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518976005833826:2058] [node 5] ICS10 traffic start
2025-11-11T17:20:36.678266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:36.725802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518971710866430:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:36.725890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:36.738636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518971710866430:2049] [node 3] ICP22 created new session: [2:7571518976005833828:2049]
2025-11-11T17:20:36.753866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518976005833828:2049] [node 3] ICS09 handshake done sender: [2:7571518976005833803:2115] self: [2:7571518976005833802:0] peer: [3:7571518975319628090:0] socket: 515 qp: -1
2025-11-11T17:20:36.779496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518976005833828:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:36.790544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:36.822991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518971710866430:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(109843675ns) {HandleHandshakeStatus:329(109839529ns) {DropIncomingHandshake:441(31007ns) DropOutgoingHandshake:456(603ns) SwitchToState:178(491ns) ProcessPendingSessionEvents:495(11932088ns) {ForwardSessionEventToSession:612(11872075ns) ForwardSessionEventToSession:612(16011ns) ForwardSessionEventToSession:612(2847ns) ForwardSessionEventToSession:612(7631ns) ForwardSessionEventToSession:612(6074ns) ForwardSessionEventToSession:612(2546ns) ForwardSessionEventToSession:612(5523ns) ForwardSessionEventToSession:612(2513ns) ForwardSessionEventToSession:612(2118ns) ForwardSessionEventToSession:612(5377ns)}}}
2025-11-11T17:20:36.886613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518971113784732:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:36.886679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:36.886725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518971113784732:2050] [node 3] ICP22 created new session: [6:7571518975408752129:2050]
2025-11-11T17:20:36.886770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518975408752129:2050] [node 3] ICS09 handshake done sender: [6:7571518975408752121:2122] self: [6:7571518975408752125:0] peer: [3:7571518975319628080:0] socket: 535 qp: -1
2025-11-11T17:20:36.886809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518975408752129:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:36.886853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:36.937453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:36.937754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518976005833822:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-11-11T17:20:36.937798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518971710866429:2048] [node 1] ICP30 unregister session Session# [2:7571518976005833822:2048] VirtualId# [2:7571518976005833820:0]
2025-11-11T17:20:36.937837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518976005833822:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:36.937941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:36.937989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:36.941759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:36.942019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518975319628089:2048] [node 1] ICS01 socket: 498 reason# EndOfStream
2025-11-11T17:20:36.942061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518971024660703:2048] [node 1] ICP30 unregister session Session# [3:7571518975319628089:2048] VirtualId# [3:7571518975319628083:0]
2025-11-11T17:20:36.942096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518975319628089:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:36.942193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:36.942247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:36.945762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:36.946427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:36.946609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518975408752116:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-11T17:20:36.946640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518971113784730:2048] [node 1] ICP30 unregister session Session# [6:7571518975408752116:2048] VirtualId# [6:7571518975408752106:0]
2025-11-11T17:20:36.946664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518975408752116:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:36.946727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:36.946784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:36.945987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518976240030782:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-11-11T17:20:36.946028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518971945063400:2048] [node 1] ICP30 unregister session Session# [5:7571518976240030782:2048] VirtualId# [5:7571518976240030776:0]
2025-11-11T17:20:36.946063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518976240030782:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:36.946151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:36.946198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:36.949960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:36.950182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518977453023160:2048] [node 1] ICS01 socket: 527 reason# EndOfStream
2025-11-11T17:20:36.950218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518973158055766:2048] [node 1] ICP30 unregister session Session# [4:7571518977453023160:2048] VirtualId# [4:7571518977453023158:0]
2025-11-11T17:20:36.950252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518977453023160:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:36.950361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:36.950407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:37.101763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:37.102000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518975319628098:2049] [node 2] ICS01 socket: 519 reason# EndOfStream
2025-11-11T17:20:37.102045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518971024660704:2049] [node 2] ICP30 unregister session Session# [3:7571518975319628098:2049] VirtualId# [3:7571518975319628090:0]
2025-11-11T17:20:37.102084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518975319628098:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.102196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:37.102256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:37.113961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:37.114189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518977453023146:2049] [node 2] ICS01 socket: 485 reason# EndOfStream
2025-11-11T17:20:37.114239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518973158055767:2049] [node 2] ICP30 unregister session Session# [4:7571518977453023146:2049] VirtualId# [4:7571518977453023139:0]
2025-11-11T17:20:37.114277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518977453023146:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.114388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:37.114431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:37.113862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:37.114245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518976240030791:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-11T17:20:37.114293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518971945063401:2049] [node 2] ICP30 unregister session Session# [5:7571518976240030791:2049] VirtualId# [5:7571518976240030780:0]
2025-11-11T17:20:37.114329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518976240030791:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.114419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:37.114466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:37.115095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:37.115333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518975408752114:2049] [node 2] ICS07 socket disconnect 487 reason# EndOfStream
2025-11-11T17:20:37.115365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518975408752114:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.115449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:37.115496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518975408752114:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:37.116606Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Couldn't connect to any resolved address
2025-11-11T17:20:37.116861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518971113784731:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:37.116908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518975408752114:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:37.116945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518971113784731:2049] [node 2] ICP30 unregister session Session# [6:7571518975408752114:2049] VirtualId# [6:7571518975408752103:0]
2025-11-11T17:20:37.116991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:37.117040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518971113784731:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:37.117055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:37.133804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518971113784731:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:37.205767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:37.205773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:37.206031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518976240030797:2050] [node 3] ICS01 socket: 508 reason# EndOfStream
2025-11-11T17:20:37.206074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518971945063402:2050] [node 3] ICP30 unregister session Session# [5:7571518976240030797:2050] VirtualId# [5:7571518976240030781:0]
2025-11-11T17:20:37.206114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518976240030797:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.206201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:37.206244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:37.206556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518975408752129:2050] [node 3] ICS01 socket: 535 reason# EndOfStream
2025-11-11T17:20:37.206592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518971113784732:2050] [node 3] ICP30 unregister session Session# [6:7571518975408752129:2050] VirtualId# [6:7571518975408752125:0]
2025-11-11T17:20:37.206622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518975408752129:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.206677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:37.206716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:37.205861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:37.206057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518977453023156:2050] [node 3] ICS01 socket: 509 reason# EndOfStream
2025-11-11T17:20:37.206091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518973158055768:2050] [node 3] ICP30 unregister session Session# [4:7571518977453023156:2050] VirtualId# [4:7571518977453023144:0]
2025-11-11T17:20:37.206120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518977453023156:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.206203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:37.206245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:37.317757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:37.318003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518976240030795:2066] [node 4] ICS01 socket: 536 reason# EndOfStream
2025-11-11T17:20:37.318051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518976240030723:2066] [node 4] ICP30 unregister session Session# [5:7571518976240030795:2066] VirtualId# [5:7571518976240030793:0]
2025-11-11T17:20:37.318086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518976240030795:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.318158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:37.318215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:37.317757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:37.318002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518975408752123:2065] [node 4] ICS07 socket disconnect 486 reason# EndOfStream
2025-11-11T17:20:37.318030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518975408752123:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.318119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:37.318160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518975408752123:2065] [node 4] ICS15 start handshake
2025-11-11T17:20:37.325898Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61418) Couldn't connect to any resolved address
2025-11-11T17:20:37.342121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518975408752080:2065] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61418) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:37.342207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518975408752123:2065] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:37.342260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518975408752080:2065] [node 4] ICP30 unregister session Session# [6:7571518975408752123:2065] VirtualId# [6:7571518975408752112:0]
2025-11-11T17:20:37.342316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:37.342353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518975408752080:2065] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61418) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:37.342369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61418) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:37.342713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [6:7571518975408752080:2065] [node 4] ICP35 event processing took too much time StateWork:293(16133189ns) {HandleHandshakeStatus:395(16129281ns) {LogHandshakeFail:472(11521308ns) {UpdateErrorStateLog:544(6663ns)} DropOutgoingHandshake:456(2536ns) ProcessPendingSessionEvents:495(1319ns) UnregisterSession:538(57802ns) {ProcessPendingSessionEvents:495(94ns) SwitchToInitialState:201(1839ns) {SwitchToState:178(907ns)}} TransitToErrorState:752(58288ns) {SwitchToState:178(8084ns) ProcessPendingSessionEvents:495(33ns) DropHandshakes:469(2229ns) {DropIncomingHandshake:441(942ns) DropOutgoingHandshake:456(138ns)}}}}
2025-11-11T17:20:37.353741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518975408752080:2065] [node 4] ICP33 wake up from error state
2025-11-11T17:20:37.406604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:37.406852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518975408752127:2062] [node 5] ICS07 socket disconnect 502 reason# EndOfStream
2025-11-11T17:20:37.406882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518975408752127:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:37.407019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:37.407065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518975408752127:2062] [node 5] ICS15 start handshake
2025-11-11T17:20:37.407779Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21236) Couldn't connect to any resolved address
2025-11-11T17:20:37.408041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518975408752096:2062] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:37.408100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518975408752127:2062] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:37.408138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518975408752096:2062] [node 5] ICP30 unregister session Session# [6:7571518975408752127:2062] VirtualId# [6:7571518975408752113:0]
2025-11-11T17:20:37.408183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:20:37.408212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518975408752096:2062] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:37.408225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:37.421734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518975408752096:2062] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:37.612478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518981742605904:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:37.612565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518981742605905:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:37.666894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518979706585156:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:37.666937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518979706585157:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:37.778000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518981811587468:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:37.778071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518981811587469:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:37.841948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518980270748498:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:37.841994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518980270748499:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:37.842032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518980270748500:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:37.902273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518980723863855:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:37.902333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518980723863856:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:37.902360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518980723863857:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:37.957683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518977733836335:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:37.957736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518977733836336:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:37.957767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518977733836337:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:38.143138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518984001552476:2070] [node 6] ICP01 ready to work
2025-11-11T17:20:38.144081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518982028803647:2062] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.203076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518986106554795:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1656
2025-11-11T17:20:38.209639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518986037573261:2106] [node 4] ICP01 ready to work
2025-11-11T17:20:38.226375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518984001552496:2071] [node 5] ICP01 ready to work
2025-11-11T17:20:38.246197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518979706585156:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(48647848ns) {Configure:104(48642085ns) {ProcessConfigured:135(31484611ns) {StartInitialHandshake:157(31481088ns) {DropHandshakes:469(3205ns) {DropIncomingHandshake:441(689ns) DropOutgoingHandshake:456(581ns)} GenerateSessionVirtualId:988(622ns) PrepareNewSessionHandshake:476(4538ns) {SwitchToState:178(2379ns)}}}}}
2025-11-11T17:20:38.247624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518986037573264:2103] [node 6] ICP01 ready to work
2025-11-11T17:20:38.259814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518986106554803:2066] [node 4] ICP01 ready to work
2025-11-11T17:20:38.258453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518977733836336:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(43290864ns) {Configure:104(43285832ns) {ProcessConfigured:135(21071062ns) {StartInitialHandshake:157(21067475ns) {DropHandshakes:469(2744ns) {DropIncomingHandshake:441(673ns) DropOutgoingHandshake:456(422ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:476(4751ns) {SwitchToState:178(2943ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.324100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518984565715825:2075] [node 6] ICP01 ready to work
2025-11-11T17:20:38.315468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518984001552496:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(88778812ns) {Configure:104(88772402ns) {ProcessConfigured:135(9475264ns) {StartInitialHandshake:157(9471943ns) {DropHandshakes:469(3222ns) {DropIncomingHandshake:441(739ns) DropOutgoingHandshake:456(760ns)} GenerateSessionVirtualId:988(5801ns) PrepareNewSessionHandshake:476(5992ns) {SwitchToState:178(3267ns)}}}}}
2025-11-11T17:20:38.325639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518985018831177:2069] [node 4] ICP01 ready to work
2025-11-11T17:20:38.265655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518986037573265:2104] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.259491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518982028803655:2064] [node 4] ICP01 ready to work
2025-11-11T17:20:38.342395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518985018831179:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:38.299986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518986106554806:2084] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.351547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518981742605905:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(141628701ns) {Configure:104(141620053ns) {ProcessConfigured:135(84943ns) {StartInitialHandshake:157(82275ns) {DropHandshakes:469(3884ns) {DropIncomingHandshake:441(822ns) DropOutgoingHandshake:456(637ns)} GenerateSessionVirtualId:988(1173ns) PrepareNewSessionHandshake:476(2459ns) {SwitchToState:178(878ns)}}}}}
2025-11-11T17:20:38.328282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518981811587469:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(79182111ns) {Configure:104(79174508ns) {ProcessConfigured:135(89528ns) {StartInitialHandshake:157(86945ns) {DropHandshakes:469(3245ns) {DropIncomingHandshake:441(690ns) DropOutgoingHandshake:456(529ns)} GenerateSessionVirtualId:988(1485ns) PrepareNewSessionHandshake:476(2446ns) {SwitchToState:178(985ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.334302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518984001552526:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34294
2025-11-11T17:20:38.295826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518982028803647:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(151432622ns) {Configure:104(151428000ns) {ProcessConfigured:135(21738061ns) {StartInitialHandshake:157(21735149ns) {DropHandshakes:469(3326ns) {DropIncomingHandshake:441(578ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:476(4925ns) {SwitchToState:178(2453ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.409877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518984565715840:2074] [node 5] ICP01 ready to work
2025-11-11T17:20:38.462505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518985018831177:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16291802ns) {Configure:104(16285843ns) {ProcessConfigured:135(98090ns) {StartInitialHandshake:157(96132ns) {DropHandshakes:469(2733ns) {DropIncomingHandshake:441(710ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:476(1677ns) {SwitchToState:178(673ns)}}}}}
2025-11-11T17:20:38.476499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831212:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.476713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831213:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.476783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831214:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.325667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518984001552511:2072] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.389005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518986106554832:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1656
2025-11-11T17:20:38.444286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518980270748499:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(209713618ns) {Configure:104(209708632ns) {ProcessConfigured:135(69396ns) {StartInitialHandshake:157(67257ns) {DropHandshakes:469(2782ns) {DropIncomingHandshake:441(727ns) DropOutgoingHandshake:456(378ns)} GenerateSessionVirtualId:988(1025ns) PrepareNewSessionHandshake:476(2595ns) {SwitchToState:178(967ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.523715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518980270748500:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(315870837ns) {Configure:104(315864914ns) {ProcessConfigured:135(75168ns) {StartInitialHandshake:157(72950ns) {DropHandshakes:469(2505ns) {DropIncomingHandshake:441(584ns) DropOutgoingHandshake:456(553ns)} GenerateSessionVirtualId:988(945ns) PrepareNewSessionHandshake:476(1856ns) {SwitchToState:178(563ns)}}}}}
2025-11-11T17:20:38.524094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518984565715864:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:38.403930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518986037573272:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45385
2025-11-11T17:20:38.410173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518986106554830:2070] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.492281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518980723863855:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(263890142ns) {Configure:104(263885158ns) {ProcessConfigured:135(82834ns) {StartInitialHandshake:157(80493ns) {DropHandshakes:469(2892ns) {DropIncomingHandshake:441(514ns) DropOutgoingHandshake:456(654ns)} GenerateSessionVirtualId:988(1186ns) PrepareNewSessionHandshake:476(1910ns) {SwitchToState:178(580ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.450328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518984001552528:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34294
2025-11-11T17:20:38.484155Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518986106554795:2078] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.411015Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573272:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.512395Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518985018831214:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.463118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518984001552529:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34294
2025-11-11T17:20:38.585646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552484:2077] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.585697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518984001552484:2077] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.594918Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552518:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.594966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518984001552518:2105] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.612481Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552526:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.612617Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552528:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.612838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518984001552530:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34294
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.498807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518986037573273:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45385
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.540459Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573267:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.540525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518986037573267:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.560159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518981742605904:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(323892296ns) {Configure:104(323883222ns) {ProcessConfigured:135(76556ns) {StartInitialHandshake:157(73443ns) {DropHandshakes:469(3018ns) {DropIncomingHandshake:441(603ns) DropOutgoingHandshake:456(727ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:476(1993ns) {SwitchToState:178(942ns)}}}}}
2025-11-11T17:20:38.560432Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573273:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.560903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518986037573276:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45385
2025-11-11T17:20:38.579376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518986037573264:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(256413901ns) {Configure:104(256407426ns) {ProcessConfigured:135(74902ns) {StartInitialHandshake:157(71806ns) {DropHandshakes:469(2904ns) {DropIncomingHandshake:441(678ns) DropOutgoingHandshake:456(584ns)} GenerateSessionVirtualId:988(711ns) PrepareNewSessionHandshake:476(1941ns) {SwitchToState:178(580ns)}}}}}
2025-11-11T17:20:38.582816Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573270:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.582864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518986037573270:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.595160Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518986037573267:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:38.551725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518986106554832:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.622017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518977733836337:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(159780964ns) {Configure:104(159774445ns) {ProcessConfigured:135(99102ns) {StartInitialHandshake:157(96718ns) {DropHandshakes:469(2760ns) {DropIncomingHandshake:441(890ns) DropOutgoingHandshake:456(360ns)} GenerateSessionVirtualId:988(1101ns) PrepareNewSessionHandshake:476(1561ns) {SwitchToState:178(583ns)}}}}}
2025-11-11T17:20:38.613080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518984001552531:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34294
2025-11-11T17:20:38.512495Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518985018831212:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.603851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518984565715864:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.674224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518986037573261:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(400321469ns) {Configure:104(400313392ns) {ProcessConfigured:135(83494ns) {StartInitialHandshake:157(81128ns) {DropHandshakes:469(2653ns) {DropIncomingHandshake:441(574ns) DropOutgoingHandshake:456(558ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:476(1959ns) {SwitchToState:178(587ns)}}}}}
2025-11-11T17:20:38.652733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518982028803655:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(291888981ns) {Configure:104(291876358ns) {ProcessConfigured:135(26198008ns) {StartInitialHandshake:157(26195743ns) {DropHandshakes:469(723ns) {DropIncomingHandshake:441(206ns) DropOutgoingHandshake:456(189ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:476(4085ns) {SwitchToState:178(1631ns)}}}}}
2025-11-11T17:20:38.626990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552529:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.627469Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518984001552484:2077] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:38.627820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518979706585157:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.627886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:38.627939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518979706585157:2049] [node 3] ICP22 created new session: [2:7571518984001552534:2049]
2025-11-11T17:20:38.627991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518984001552534:2049] [node 3] ICS09 handshake done sender: [2:7571518984001552484:2077] self: [2:7571518984001552483:0] peer: [3:7571518986106554837:0] socket: 480 qp: -1
2025-11-11T17:20:38.628028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518984001552534:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:38.628060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:38.638019Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552530:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.657846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [2:7571518979706585157:2049] [node 3] ICP35 event processing took too much time StateWork:293(19621547ns)
2025-11-11T17:20:38.675989Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518984001552518:2105] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:38.676249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518984001552496:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.676310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:38.676366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518984001552496:2071] [node 5] ICP22 created new session: [2:7571518984001552539:2071]
2025-11-11T17:20:38.676431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518984001552539:2071] [node 5] ICS09 handshake done sender: [2:7571518984001552518:2105] self: [2:7571518984001552512:0] peer: [5:7571518985018831228:0] socket: 487 qp: -1
2025-11-11T17:20:38.676473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518984001552539:2071] [node 5] ICS10 traffic start
2025-11-11T17:20:38.676510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:38.682496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518984001552476:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(514924045ns) {Configure:104(514910367ns) {ProcessConfigured:135(12157581ns) {StartInitialHandshake:157(12155265ns) {DropHandshakes:469(3017ns) {DropIncomingHandshake:441(572ns) DropOutgoingHandshake:456(941ns)} GenerateSessionVirtualId:988(770ns) PrepareNewSessionHandshake:476(4696ns) {SwitchToState:178(1577ns)}}}}}
2025-11-11T17:20:38.683567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518984565715840:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(182235289ns) {Configure:104(182224164ns) {ProcessConfigured:135(17645170ns) {StartInitialHandshake:157(17642329ns) {DropHandshakes:469(2802ns) {DropIncomingHandshake:441(632ns) DropOutgoingHandshake:456(404ns)} GenerateSessionVirtualId:988(1280ns) PrepareNewSessionHandshake:476(3551ns) {SwitchToState:178(1704ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.701150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803707:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:38.551899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518981811587469:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.706517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518977733836335:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(444813098ns) {Configure:104(444808677ns) {ProcessConfigured:135(84842ns) {StartInitialHandshake:157(81499ns) {DropHandshakes:469(2953ns) {DropIncomingHandshake:441(706ns) DropOutgoingHandshake:456(639ns)} GenerateSessionVirtualId:988(922ns) PrepareNewSessionHandshake:476(2226ns) {SwitchToState:178(642ns)}}}}}
2025-11-11T17:20:38.512642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518980723863855:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.525370Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518985018831213:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.718361Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552531:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.718517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518979706585156:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.552275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518986106554836:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1656
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.714367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518986037573270:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:38.547695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518980723863856:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(293460969ns) {Configure:104(293455711ns) {ProcessConfigured:135(111159ns) {StartInitialHandshake:157(107924ns) {DropHandshakes:469(3974ns) {DropIncomingHandshake:441(895ns) DropOutgoingHandshake:456(697ns)} GenerateSessionVirtualId:988(1113ns) PrepareNewSessionHandshake:476(2965ns) {SwitchToState:178(675ns)}}}}}
2025-11-11T17:20:38.757900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518984001552511:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(291013361ns) {Configure:104(291001841ns) {ProcessConfigured:135(6499101ns) {StartInitialHandshake:157(6496297ns) {DropHandshakes:469(2854ns) {DropIncomingHandshake:441(523ns) DropOutgoingHandshake:456(634ns)} GenerateSessionVirtualId:988(572ns) PrepareNewSessionHandshake:476(3804ns) {SwitchToState:178(1753ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.766251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803710:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.772677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803711:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:38.552541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518981811587468:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(290559713ns) {Configure:104(290552864ns) {ProcessConfigured:135(19370867ns) {StartInitialHandshake:157(19368175ns) {DropHandshakes:469(2625ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:476(1122ns) {SwitchToState:178(96ns)}}}}}
2025-11-11T17:20:38.730210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518986037573265:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.745752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:38.754179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518981742605905:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.774034Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573276:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.785642Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573275:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.785703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518986037573275:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.785832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518986037573265:2104] [node 5] ICP22 created new session: [1:7571518986037573286:2104]
2025-11-11T17:20:38.785942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:38.786013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518981742605905:2049] [node 3] ICP22 created new session: [1:7571518986037573288:2049]
2025-11-11T17:20:38.786061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518986037573288:2049] [node 3] ICS09 handshake done sender: [1:7571518986037573270:2111] self: [1:7571518986037573269:0] peer: [3:7571518986106554839:0] socket: 488 qp: -1
2025-11-11T17:20:38.786097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518986037573288:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:38.786141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:38.786635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518981742605905:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(32252482ns) {HandleHandshakeStatus:329(32250074ns) {DropIncomingHandshake:441(408ns) DropOutgoingHandshake:456(810ns) SwitchToState:178(512ns) ProcessPendingSessionEvents:495(104827ns) {ForwardSessionEventToSession:612(61461ns) ForwardSessionEventToSession:612(2931ns) ForwardSessionEventToSession:612(2393ns) ForwardSessionEventToSession:612(5168ns) ForwardSessionEventToSession:612(2984ns) ForwardSessionEventToSession:612(6920ns) ForwardSessionEventToSession:612(2218ns) ForwardSessionEventToSession:612(2762ns) ForwardSessionEventToSession:612(4510ns) ForwardSessionEventToSession:612(2601ns)}}}
2025-11-11T17:20:38.786918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518981742605905:2049] [node 3] ICP18 handshake (actor [1:7571518986037573276:2115]) is too old
2025-11-11T17:20:38.787646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518986037573286:2104] [node 5] ICS09 handshake done sender: [1:7571518986037573267:2110] self: [1:7571518986037573266:0] peer: [5:7571518985018831225:0] socket: 484 qp: -1
2025-11-11T17:20:38.547960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518980723863856:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.552655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518981811587468:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.811388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518984001552549:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34294
2025-11-11T17:20:38.815150Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518984001552531:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:38.815442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518979706585156:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.815545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:38.815607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518979706585156:2048] [node 1] ICP22 created new session: [2:7571518984001552550:2048]
2025-11-11T17:20:38.815657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518984001552550:2048] [node 1] ICS09 handshake done sender: [2:7571518984001552531:2117] self: [2:7571518984001552544:0] peer: [1:7571518986037573274:0] socket: 510 qp: -1
2025-11-11T17:20:38.815691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518984001552550:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:38.815749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:38.834250Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552541:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.834293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518984001552541:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.835308Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518984001552541:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:38.835614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518984001552476:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.835685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:38.835736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518984001552476:2070] [node 6] ICP22 created new session: [2:7571518984001552553:2070]
2025-11-11T17:20:38.835783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518984001552553:2070] [node 6] ICS09 handshake done sender: [2:7571518984001552541:2118] self: [2:7571518984001552536:0] peer: [6:7571518982028803715:0] socket: 504 qp: -1
2025-11-11T17:20:38.835817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518984001552553:2070] [node 6] ICS10 traffic start
2025-11-11T17:20:38.835851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.785724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518982028803707:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.806649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518986037573286:2104] [node 5] ICS10 traffic start
2025-11-11T17:20:38.814048Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518986037573275:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:38.838801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518981742605904:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.838868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:38.838927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518981742605904:2048] [node 2] ICP22 created new session: [1:7571518986037573293:2048]
2025-11-11T17:20:38.838996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518986037573293:2048] [node 2] ICS09 handshake done sender: [1:7571518986037573275:2114] self: [1:7571518986037573274:0] peer: [2:7571518984001552544:0] socket: 508 qp: -1
2025-11-11T17:20:38.839037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518986037573293:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:38.839070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:38.844661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:38.582026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518985018831179:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(159013700ns) {Configure:104(159007264ns) {ProcessConfigured:135(15988315ns) {StartInitialHandshake:157(15984976ns) {DropHandshakes:469(2291ns) {DropIncomingHandshake:441(497ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(1867ns) PrepareNewSessionHandshake:476(6517ns) {SwitchToState:178(3160ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.829064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518984565715876:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:38.586387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518986106554806:2084] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(243775530ns) {Configure:104(243765760ns) {ProcessConfigured:135(15511018ns) {StartInitialHandshake:157(15508710ns) {DropHandshakes:469(1655ns) {DropIncomingHandshake:441(432ns) DropOutgoingHandshake:456(285ns)} GenerateSessionVirtualId:988(790ns) PrepareNewSessionHandshake:476(1960ns) {SwitchToState:178(361ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.872394Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573282:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.796154Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518982028803711:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.629290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831233:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.677953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.683346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518980723863857:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(474958615ns) {Configure:104(474948325ns) {ProcessConfigured:135(41406257ns) {StartInitialHandshake:157(41403878ns) {DropHandshakes:469(2835ns) {DropIncomingHandshake:441(470ns) DropOutgoingHandshake:456(616ns)} GenerateSessionVirtualId:988(1107ns) PrepareNewSessionHandshake:476(2414ns) {SwitchToState:178(1359ns)}}}}}
2025-11-11T17:20:38.725389Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:38.737839Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518985018831212:2107] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:38.738120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518980723863856:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518985018831214:2109] held: no
2025-11-11T17:20:38.738334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518985018831234:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.738517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831238:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.738618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518980723863857:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.769864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518980723863855:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.769927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:38.769985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518980723863855:2048] [node 1] ICP22 created new session: [5:7571518985018831240:2048]
2025-11-11T17:20:38.770034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518985018831240:2048] [node 1] ICS09 handshake done sender: [5:7571518985018831212:2107] self: [5:7571518985018831225:0] peer: [1:7571518986037573266:0] socket: 493 qp: -1
2025-11-11T17:20:38.770076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518985018831240:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:38.770120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:38.776115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831237:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.793968Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518985018831214:2109] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:38.795256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518980723863856:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.795334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:38.795382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518980723863856:2049] [node 2] ICP22 created new session: [5:7571518985018831245:2049]
2025-11-11T17:20:38.795427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518985018831245:2049] [node 2] ICS09 handshake done sender: [5:7571518985018831214:2109] self: [5:7571518985018831228:0] peer: [2:7571518984001552512:0] socket: 495 qp: -1
2025-11-11T17:20:38.795459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518985018831245:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:38.795510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:38.796545Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518985018831231:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.796568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518985018831231:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.798191Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518985018831238:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.803327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518985018831177:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.806509Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518985018831231:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:38.829170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831243:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.850000Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518985018831234:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:38.852174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518980723863857:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.852262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:38.852320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518980723863857:2050] [node 3] ICP22 created new session: [5:7571518985018831249:2050]
2025-11-11T17:20:38.852375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518985018831249:2050] [node 3] ICS09 handshake done sender: [5:7571518985018831234:2118] self: [5:7571518985018831239:0] peer: [3:7571518986106554841:0] socket: 517 qp: -1
2025-11-11T17:20:38.852410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518985018831249:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:38.852471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:38.857896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518985018831179:2067] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.845676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518984565715825:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(387583473ns) {Configure:104(387579040ns) {ProcessConfigured:135(17755973ns) {StartInitialHandshake:157(17750396ns) {DropHandshakes:469(2602ns) {DropIncomingHandshake:441(511ns) DropOutgoingHandshake:456(586ns)} GenerateSessionVirtualId:988(1246ns) PrepareNewSessionHandshake:476(3245ns) {SwitchToState:178(94ns)}}}}}
2025-11-11T17:20:38.641927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518986106554836:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.872457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518986037573282:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.796283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518982028803647:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.889660Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518984001552546:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.889741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518984001552546:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.920226Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518984001552546:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.887098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.650344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518986106554844:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1656
2025-11-11T17:20:38.853785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518984565715869:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.853828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518984565715869:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.857734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518980270748498:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(472533616ns) {Configure:104(472526801ns) {ProcessConfigured:135(28029186ns) {StartInitialHandshake:157(28025596ns) {DropHandshakes:469(3091ns) {DropIncomingHandshake:441(967ns) DropOutgoingHandshake:456(556ns)} GenerateSessionVirtualId:988(1019ns) PrepareNewSessionHandshake:476(3975ns) {SwitchToState:178(1575ns)}}}}}
2025-11-11T17:20:38.862300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518984565715876:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.862444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518980270748499:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.865901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518984565715877:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:38.890160Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518984565715869:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:38.914268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518984565715840:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.914337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:38.914392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518984565715840:2074] [node 5] ICP22 created new session: [4:7571518984565715881:2074]
2025-11-11T17:20:38.914436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518984565715881:2074] [node 5] ICS09 handshake done sender: [4:7571518984565715869:2115] self: [4:7571518984565715868:0] peer: [5:7571518985018831248:0] socket: 521 qp: -1
2025-11-11T17:20:38.914469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518984565715881:2074] [node 5] ICS10 traffic start
2025-11-11T17:20:38.914528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:38.920070Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518984565715877:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.935390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518980270748498:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.935521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518984565715883:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:38.940760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518984565715884:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:38.797894Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518982028803710:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.882674Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518986037573282:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:38.956517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518984001552511:2072] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.956616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:38.956672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518984001552511:2072] [node 4] ICP22 created new session: [2:7571518984001552556:2072]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.887172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518985018831179:2067] [node 6] ICP22 created new session: [5:7571518985018831251:2067]
2025-11-11T17:20:38.650402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518986106554845:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1656
2025-11-11T17:20:38.882980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518986037573264:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.949973Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518984565715876:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:38.901978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518985018831252:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42514
2025-11-11T17:20:38.913862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518985018831251:2067] [node 6] ICS09 handshake done sender: [5:7571518985018831231:2116] self: [5:7571518985018831230:0] peer: [6:7571518982028803712:0] socket: 503 qp: -1
2025-11-11T17:20:38.915903Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518985018831238:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:38.916131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518985018831177:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.916179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:38.916227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518985018831177:2069] [node 4] ICP22 created new session: [5:7571518985018831253:2069]
2025-11-11T17:20:38.916278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518985018831253:2069] [node 4] ICS09 handshake done sender: [5:7571518985018831238:2121] self: [5:7571518985018831248:0] peer: [4:7571518984565715868:0] socket: 523 qp: -1
2025-11-11T17:20:38.916309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518985018831253:2069] [node 4] ICS10 traffic start
2025-11-11T17:20:38.916363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:38.929785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518985018831251:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:38.953727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.825773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518977733836336:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.883025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:38.883074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518986037573264:2103] [node 6] ICP22 created new session: [1:7571518986037573296:2103]
2025-11-11T17:20:38.883125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518986037573296:2103] [node 6] ICS09 handshake done sender: [1:7571518986037573282:2116] self: [1:7571518986037573281:0] peer: [6:7571518982028803714:0] socket: 525 qp: -1
2025-11-11T17:20:38.883160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518986037573296:2103] [node 6] ICS10 traffic start
2025-11-11T17:20:38.883227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:38.890827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571518986037573265:2104] [node 5] ICP35 event processing took too much time PendingConnection:278(160407786ns) {HandleHandshakeStatus:329(160404358ns) {DropIncomingHandshake:441(13635ns) DropOutgoingHandshake:456(714ns) SwitchToState:178(1280ns) ProcessPendingSessionEvents:495(145901ns) {ForwardSessionEventToSession:612(85650ns) ForwardSessionEventToSession:612(3876ns) ForwardSessionEventToSession:612(4157ns) ForwardSessionEventToSession:612(6236ns) ForwardSessionEventToSession:612(2691ns) ForwardSessionEventToSession:612(4044ns) ForwardSessionEventToSession:612(8803ns) ForwardSessionEventToSession:612(2912ns) ForwardSessionEventToSession:612(5276ns) ForwardSessionEventToSession:612(2681ns)}}}
2025-11-11T17:20:38.961161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518986037573298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45385
2025-11-11T17:20:38.961222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518986037573299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45385
2025-11-11T17:20:38.969679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573285:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.969735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518986037573285:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.650574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518986106554846:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1656
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.825946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518977733836336:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(36003910ns) {IncomingHandshake:261(36001715ns) {DropIncomingHandshake:441(478ns) PrepareNewSessionHandshake:476(8726022ns) {SwitchToState:178(2136ns)} GenerateSessionVirtualId:988(839ns) IssueIncomingHandshakeReply:193(16128013ns) {SwitchToState:178(1798ns)}}}
2025-11-11T17:20:38.953741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518984565715880:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:38.967185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518984001552556:2072] [node 4] ICS09 handshake done sender: [2:7571518984001552546:2119] self: [2:7571518984001552545:0] peer: [4:7571518984565715878:0] socket: 524 qp: -1
2025-11-11T17:20:38.654187Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518986106554845:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:39.010109Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518986037573285:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:38.977841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518984001552556:2072] [node 4] ICS10 traffic start
2025-11-11T17:20:38.982293Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518984565715880:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.982355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518980270748499:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.982422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:38.982508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518980270748499:2049] [node 2] ICP22 created new session: [4:7571518984565715887:2049]
2025-11-11T17:20:38.982555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518984565715887:2049] [node 2] ICS09 handshake done sender: [4:7571518984565715876:2118] self: [4:7571518984565715878:0] peer: [2:7571518984001552545:0] socket: 527 qp: -1
2025-11-11T17:20:38.982594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518984565715887:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:38.982646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:38.983882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518980270748500:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.994139Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:39.014245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518988860683187:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:39.045722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518980270748499:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:39.050298Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518984565715877:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:38.829655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518977733836335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:39.041702Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:38.654279Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518986106554795:2078] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:39.005756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:39.078262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518980270748498:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.694128Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:38.853866Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:39.030346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518984001552511:2072] [node 4] ICP35 event processing took too much time PendingConnection:278(73580351ns) {HandleHandshakeStatus:329(73577001ns) {DropIncomingHandshake:441(22003ns) DropOutgoingHandshake:456(645ns) SwitchToState:178(869ns) ProcessPendingSessionEvents:495(148326ns) {ForwardSessionEventToSession:612(86581ns) ForwardSessionEventToSession:612(8412ns) ForwardSessionEventToSession:612(3942ns) ForwardSessionEventToSession:612(7081ns) ForwardSessionEventToSession:612(4143ns) ForwardSessionEventToSession:612(3139ns) ForwardSessionEventToSession:612(12888ns) ForwardSessionEventToSession:612(3217ns) ForwardSessionEventToSession:612(3810ns) ForwardSessionEventToSession:612(4557ns)}}}
2025-11-11T17:20:39.043377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571518986037573264:2103] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:38.859783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803716:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:39.078368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:39.046257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518986037573261:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.699746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518981811587469:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.699814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:38.699876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518981811587469:2049] [node 2] ICP22 created new session: [3:7571518986106554847:2049]
2025-11-11T17:20:38.699953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518986106554847:2049] [node 2] ICS09 handshake done sender: [3:7571518986106554795:2078] self: [3:7571518986106554837:0] peer: [2:7571518984001552483:0] socket: 481 qp: -1
2025-11-11T17:20:38.700002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518986106554847:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:38.700045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:38.700322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518981811587469:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:38.791013Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518986106554842:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.791040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518986106554842:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.791556Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518986106554832:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:38.791803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518981811587468:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.791884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:38.791935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518981811587468:2048] [node 1] ICP22 created new session: [3:7571518986106554851:2048]
2025-11-11T17:20:38.791977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518986106554851:2048] [node 1] ICS09 handshake done sender: [3:7571518986106554832:2111] self: [3:7571518986106554839:0] peer: [1:7571518986037573269:0] socket: 490 qp: -1
2025-11-11T17:20:38.792012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518986106554851:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:38.792051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:38.806191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518986106554842:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:38.858434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518986106554806:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:38.858489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:38.858544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518986106554806:2084] [node 5] ICP22 created new session: [3:7571518986106554857:2084]
2025-11-11T17:20:38.858596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518986106554857:2084] [node 5] ICS09 handshake done sender: [3:7571518986106554842:2115] self: [3:7571518986106554841:0] peer: [5:7571518985018831239:0] socket: 512 qp: -1
2025-11-11T17:20:38.858633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518986106554857:2084] [node 5] ICS10 traffic start
2025-11-11T17:20:38.858667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:38.861977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518986106554830:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(368965656ns) {Configure:104(368958229ns) {ProcessConfigured:135(11965658ns) {StartInitialHandshake:157(11962054ns) {DropHandshakes:469(3199ns) {DropIncomingHandshake:441(599ns) DropOutgoingHandshake:456(648ns)} GenerateSessionVirtualId:988(721ns) PrepareNewSessionHandshake:476(3117ns) {SwitchToState:178(1245ns)}}}}}
2025-11-11T17:20:38.877727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518986106554853:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1656
2025-11-11T17:20:38.891847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518986106554803:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(495559209ns) {Configure:104(495551011ns) {ProcessConfigured:135(78675019ns) {StartInitialHandshake:157(78670366ns) {DropHandshakes:469(2109ns) {DropIncomingHandshake:441(875ns) DropOutgoingHandshake:456(279ns)} GenerateSessionVirtualId:988(3401ns) PrepareNewSessionHandshake:476(3892ns) {SwitchToState:178(2132ns)}}}}}
2025-11-11T17:20:38.908729Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:38.908959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571518986106554806:2084] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:38.953915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518986106554859:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.953957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518986106554859:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.967067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518986106554803:2066] [node 4] ICP35 event processing took too much time PendingConnection:278(16003844ns) {IncomingHandshake:261(16001440ns) {DropIncomingHandshake:441(976ns) PrepareNewSessionHandshake:476(1816ns) {SwitchToState:178(869ns)} GenerateSessionVirtualId:988(758ns) IssueIncomingHandshakeReply:193(8331ns)}}
2025-11-11T17:20:38.984212Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518986106554854:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:38.984236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518986106554854:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:38.986074Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518986106554859:2121] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:39.002409Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518986106554854:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:39.042012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518986106554830:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:39.042080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:39.042142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518986106554830:2070] [node 6] ICP22 created new session: [3:7571518990401522161:2070]
2025-11-11T17:20:39.042195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518990401522161:2070] [node 6] ICS09 handshake done sender: [3:7571518986106554859:2121] self: [3:7571518986106554856:0] peer: [6:7571518982028803723:0] socket: 530 qp: -1
2025-11-11T17:20:39.042230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518990401522161:2070] [node 6] ICS10 traffic start
2025-11-11T17:20:39.042290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:39.042762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518986106554803:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:39.042806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:39.078430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518980270748498:2048] [node 1] ICP22 created new session: [4:7571518988860683188:2048]
2025-11-11T17:20:39.115104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571518985018831179:2067] [node 6] ICP35 event processing took too much time PendingConnection:278(210798549ns) {HandleHandshakeStatus:329(200321314ns) {DropIncomingHandshake:441(18738ns) DropOutgoingHandshake:456(4212ns) SwitchToState:178(1082ns) ProcessPendingSessionEvents:495(78657817ns) {ForwardSessionEventToSession:612(30354529ns) ForwardSessionEventToSession:612(9211ns) ForwardSessionEventToSession:612(15863189ns) ForwardSessionEventToSession:612(12209760ns) ForwardSessionEventToSession:612(4553ns) ForwardSessionEventToSession:612(7686675ns) ForwardSessionEventToSession:612(8028ns) ForwardSessionEventToSession:612(4880420ns) ForwardSessionEventToSession:612(7602510ns) ForwardSessionEventToSession:612(4781ns)}}}
2025-11-11T17:20:39.078484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518988860683188:2048] [node 1] ICS09 handshake done sender: [4:7571518984565715877:2119] self: [4:7571518984565715885:0] peer: [1:7571518986037573284:0] socket: 498 qp: -1
2025-11-11T17:20:39.078525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518988860683188:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:39.078563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:39.125755Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:39.129842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518980270748500:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571518984565715880:2120] held: no
2025-11-11T17:20:39.133797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518984565715890:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48396
2025-11-11T17:20:39.046296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:38.869686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803713:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:39.046353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518986037573261:2106] [node 4] ICP22 created new session: [1:7571518990332540597:2106]
2025-11-11T17:20:39.046396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518990332540597:2106] [node 4] ICS09 handshake done sender: [1:7571518986037573285:2117] self: [1:7571518986037573284:0] peer: [4:7571518984565715885:0] socket: 529 qp: -1
2025-11-11T17:20:38.871886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518977733836336:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518982028803707:2114] held: no
2025-11-11T17:20:39.046428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518990332540597:2106] [node 4] ICS10 traffic start
2025-11-11T17:20:38.886433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803717:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:38.886505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803718:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:39.046470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:38.886898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518977733836335:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(31187476ns) {IncomingHandshake:261(31185662ns) {DropIncomingHandshake:441(263ns) PrepareNewSessionHandshake:476(1536ns) {SwitchToState:178(837ns)} GenerateSessionVirtualId:988(593ns) IssueIncomingHandshakeReply:193(4018179ns) {SwitchToState:178(2261ns)}}}
2025-11-11T17:20:38.907511Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518982028803711:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:38.913974Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518982028803710:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:38.918010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518982028803647:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.918056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:38.918112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518982028803647:2062] [node 5] ICP22 created new session: [6:7571518982028803720:2062]
2025-11-11T17:20:38.918164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518982028803720:2062] [node 5] ICS09 handshake done sender: [6:7571518982028803711:2117] self: [6:7571518982028803712:0] peer: [5:7571518985018831230:0] socket: 528 qp: -1
2025-11-11T17:20:39.042846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518986106554803:2066] [node 4] ICP22 created new session: [3:7571518990401522163:2066]
2025-11-11T17:20:38.918199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518982028803720:2062] [node 5] ICS10 traffic start
2025-11-11T17:20:38.918229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:39.042889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518990401522163:2066] [node 4] ICS09 handshake done sender: [3:7571518986106554854:2120] self: [3:7571518986106554849:0] peer: [4:7571518984565715889:0] socket: 507 qp: -1
2025-11-11T17:20:38.934081Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518982028803717:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:39.042919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518990401522163:2066] [node 4] ICS10 traffic start
2025-11-11T17:20:38.934617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518977733836337:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:38.941507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518977733836336:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20824038ns) {HandleHandshakeStatus:395(20821097ns) {LogHandshakeFail:472(15510ns) {UpdateErrorStateLog:544(4857ns)} DropOutgoingHandshake:456(5432ns) ProcessPendingSessionEvents:495(2935998ns) {DropSessionEvent:512(12446ns) DropSessionEvent:512(5335ns) DropSessionEvent:512(6289ns) DropSessionEvent:512(5722ns) DropSessionEvent:512(4899ns) DropSessionEvent:512(5079ns) DropSessionEvent:512(22206ns) DropSessionEvent:512(6039ns) DropSessionEvent:512(5544ns) DropSessionEvent:512(5102ns)}}}
2025-11-11T17:20:39.042944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:38.953841Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518982028803707:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:38.953964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518977733836335:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.965818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:38.973929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518977733836335:2048] [node 1] ICP22 created new session: [6:7571518982028803724:2048]
2025-11-11T17:20:38.975173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518977733836336:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:38.975220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:38.975286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518977733836336:2049] [node 2] ICP22 created new session: [6:7571518982028803726:2049]
2025-11-11T17:20:38.975339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518982028803726:2049] [node 2] ICS09 handshake done sender: [6:7571518982028803707:2114] self: [6:7571518982028803715:0] peer: [2:7571518984001552536:0] socket: 520 qp: -1
2025-11-11T17:20:38.975379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518982028803726:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:38.975416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:38.983515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518982028803725:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:39.022151Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518982028803717:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:39.040624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518977733836337:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:39.045906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518982028803724:2048] [node 1] ICS09 handshake done sender: [6:7571518982028803710:2116] self: [6:7571518982028803714:0] peer: [1:7571518986037573281:0] socket: 526 qp: -1
2025-11-11T17:20:39.053794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518982028803724:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:39.053849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:39.059653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:39.069738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518977733836337:2050] [node 3] ICP22 created new session: [6:7571518986323771024:2050]
2025-11-11T17:20:39.093763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518986323771024:2050] [node 3] ICS09 handshake done sender: [6:7571518982028803717:2120] self: [6:7571518982028803723:0] peer: [3:7571518986106554856:0] socket: 538 qp: -1
2025-11-11T17:20:39.094206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518986323771026:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:39.105823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518986323771024:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:39.105900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:39.134200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518977733836337:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(100301167ns) {HandleHandshakeStatus:329(100297069ns) {DropIncomingHandshake:441(3999ns) DropOutgoingHandshake:456(21294ns) SwitchToState:178(8279ns) ProcessPendingSessionEvents:495(154524ns) {ForwardSessionEventToSession:612(91093ns) ForwardSessionEventToSession:612(3796ns) ForwardSessionEventToSession:612(6057ns) ForwardSessionEventToSession:612(3633ns) ForwardSessionEventToSession:612(6213ns) ForwardSessionEventToSession:612(6171ns) ForwardSessionEventToSession:612(4239ns) ForwardSessionEventToSession:612(14720ns) ForwardSessionEventToSession:612(6245ns) ForwardSessionEventToSession:612(2926ns)}}}
2025-11-11T17:20:39.142005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518977733836335:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(192697467ns) {HandleHandshakeStatus:329(192693998ns) {DropIncomingHandshake:441(1545ns) DropOutgoingHandshake:456(4669ns) SwitchToState:178(5209ns) ProcessPendingSessionEvents:495(56069260ns) {ForwardSessionEventToSession:612(24033000ns) ForwardSessionEventToSession:612(8864ns) ForwardSessionEventToSession:612(8004592ns) ForwardSessionEventToSession:612(10661ns) ForwardSessionEventToSession:612(11968683ns) ForwardSessionEventToSession:612(4066966ns) ForwardSessionEventToSession:612(3989ns) ForwardSessionEventToSession:612(3975ns) ForwardSessionEventToSession:612(7926930ns) ForwardSessionEventToSession:612(3691ns)}}}
2025-11-11T17:20:39.158515Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518984565715880:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:39.159022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518980270748500:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:39.159099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:39.159152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518980270748500:2050] [node 3] ICP22 created new session: [4:7571518988860683191:2050]
2025-11-11T17:20:39.159206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518988860683191:2050] [node 3] ICS09 handshake done sender: [4:7571518984565715880:2120] self: [4:7571518984565715889:0] peer: [3:7571518986106554849:0] socket: 489 qp: -1
2025-11-11T17:20:39.159245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518988860683191:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:39.159285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:39.226302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:728: Handshake [6:7571518982028803706:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-11T17:20:39.231568Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518984565715884:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:39.282496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:728: Handshake [4:7571518984565715875:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-11T17:20:39.283355Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518986037573299:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:39.283586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518986037573261:2106] [node 4] ICP18 handshake (actor [1:7571518986037573299:2119]) is too old
2025-11-11T17:20:39.286442Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518986323771026:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:39.286646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518982028803655:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:39.289801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518984565715875:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:39.291068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518986323771029:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41237
2025-11-11T17:20:39.301139Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518984565715875:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:39.301876Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518986323771026:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:39.302250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518982028803655:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:39.303254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518984565715825:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:39.303618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:39.303674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518984565715825:2075] [node 6] ICP22 created new session: [4:7571518988860683195:2075]
2025-11-11T17:20:39.303741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518988860683195:2075] [node 6] ICS09 handshake done sender: [4:7571518984565715875:2117] self: [4:7571518984565715872:0] peer: [6:7571518986323771028:0] socket: 496 qp: -1
2025-11-11T17:20:39.303777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518988860683195:2075] [node 6] ICS10 traffic start
2025-11-11T17:20:39.303816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:39.302347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:39.302403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518982028803655:2064] [node 4] ICP22 created new session: [6:7571518986323771030:2064]
2025-11-11T17:20:39.302449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518986323771030:2064] [node 4] ICS09 handshake done sender: [6:7571518986323771026:2123] self: [6:7571518986323771028:0] peer: [4:7571518984565715872:0] socket: 500 qp: -1
2025-11-11T17:20:39.302487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518986323771030:2064] [node 4] ICS10 traffic start
2025-11-11T17:20:39.302529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:39.425795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:39.426038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518984001552550:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-11-11T17:20:39.426083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518979706585156:2048] [node 1] ICP30 unregister session Session# [2:7571518984001552550:2048] VirtualId# [2:7571518984001552544:0]
2025-11-11T17:20:39.426119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518984001552550:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.426236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:39.426285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:39.439203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:39.439444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518986106554851:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-11-11T17:20:39.439485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518981811587468:2048] [node 1] ICP30 unregister session Session# [3:7571518986106554851:2048] VirtualId# [3:7571518986106554839:0]
2025-11-11T17:20:39.439537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518986106554851:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.439635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:39.439683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:39.453823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:39.454122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518982028803724:2048] [node 1] ICS07 socket disconnect 526 reason# EndOfStream
2025-11-11T17:20:39.454152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518982028803724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.454277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:39.454323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518982028803724:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:39.454897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:39.455177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518985018831240:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-11T17:20:39.455232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518980723863855:2048] [node 1] ICP30 unregister session Session# [5:7571518985018831240:2048] VirtualId# [5:7571518985018831225:0]
2025-11-11T17:20:39.455285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518985018831240:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.455389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:39.455477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:39.461272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:39.462109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518988860683188:2048] [node 1] ICS01 socket: 498 reason# EndOfStream
2025-11-11T17:20:39.462164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518980270748498:2048] [node 1] ICP30 unregister session Session# [4:7571518988860683188:2048] VirtualId# [4:7571518984565715885:0]
2025-11-11T17:20:39.462204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518988860683188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.462357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:39.462414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:39.486762Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18865) Couldn't connect to any resolved address
2025-11-11T17:20:39.494791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518977733836335:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18865) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:39.509865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518982028803724:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:39.521403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518977733836335:2048] [node 1] ICP30 unregister session Session# [6:7571518982028803724:2048] VirtualId# [6:7571518982028803714:0]
2025-11-11T17:20:39.521503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:39.521543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518977733836335:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:39.521559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:39.521835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [6:7571518977733836335:2048] [node 1] ICP35 event processing took too much time StateWork:293(34539446ns) {HandleHandshakeStatus:395(34532906ns) {LogHandshakeFail:472(13737ns) {UpdateErrorStateLog:544(3648ns)} DropOutgoingHandshake:456(3230ns) ProcessPendingSessionEvents:495(913ns) UnregisterSession:538(7924697ns) {ProcessPendingSessionEvents:495(256ns) SwitchToInitialState:201(2312ns) {SwitchToState:178(1323ns)}} TransitToErrorState:752(57742ns) {SwitchToState:178(7765ns) ProcessPendingSessionEvents:495(215ns) DropHandshakes:469(3549ns) {DropIncomingHandshake:441(621ns) DropOutgoingHandshake:456(751ns)}}}}
2025-11-11T17:20:39.534069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518977733836335:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:39.573782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:39.574170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518986106554847:2049] [node 2] ICS01 socket: 481 reason# EndOfStream
2025-11-11T17:20:39.574213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518981811587469:2049] [node 2] ICP30 unregister session Session# [3:7571518986106554847:2049] VirtualId# [3:7571518986106554837:0]
2025-11-11T17:20:39.574251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518986106554847:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.574365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:39.574421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:39.579989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:39.580264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518982028803726:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-11T17:20:39.580309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518977733836336:2049] [node 2] ICP30 unregister session Session# [6:7571518982028803726:2049] VirtualId# [6:7571518982028803715:0]
2025-11-11T17:20:39.580347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518982028803726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.580444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:39.580491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:39.580920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:39.581122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518984565715887:2049] [node 2] ICS01 socket: 527 reason# EndOfStream
2025-11-11T17:20:39.581184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518980270748499:2049] [node 2] ICP30 unregister session Session# [4:7571518984565715887:2049] VirtualId# [4:7571518984565715878:0]
2025-11-11T17:20:39.581238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518984565715887:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.581340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:39.581414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:39.581744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:39.581947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518985018831245:2049] [node 2] ICS01 socket: 495 reason# EndOfStream
2025-11-11T17:20:39.581992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518980723863856:2049] [node 2] ICP30 unregister session Session# [5:7571518985018831245:2049] VirtualId# [5:7571518985018831228:0]
2025-11-11T17:20:39.582021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518985018831245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.582092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:39.582142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:39.673001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:39.677768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:39.677867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:39.678114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518986323771024:2050] [node 3] ICS01 socket: 538 reason# EndOfStream
2025-11-11T17:20:39.678155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518977733836337:2050] [node 3] ICP30 unregister session Session# [6:7571518986323771024:2050] VirtualId# [6:7571518982028803723:0]
2025-11-11T17:20:39.678195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518986323771024:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.678319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:39.678366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:39.673270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518985018831249:2050] [node 3] ICS01 socket: 517 reason# EndOfStream
2025-11-11T17:20:39.673868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518980723863857:2050] [node 3] ICP30 unregister session Session# [5:7571518985018831249:2050] VirtualId# [5:7571518985018831239:0]
2025-11-11T17:20:39.673911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518985018831249:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.674020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:39.674068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:39.682309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518988860683191:2050] [node 3] ICS01 socket: 489 reason# EndOfStream
2025-11-11T17:20:39.682389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518980270748500:2050] [node 3] ICP30 unregister session Session# [4:7571518988860683191:2050] VirtualId# [4:7571518984565715889:0]
2025-11-11T17:20:39.682426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518988860683191:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.682579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:39.682630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:39.833836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:39.834083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518986323771030:2064] [node 4] ICS01 socket: 500 reason# EndOfStream
2025-11-11T17:20:39.834123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518982028803655:2064] [node 4] ICP30 unregister session Session# [6:7571518986323771030:2064] VirtualId# [6:7571518986323771028:0]
2025-11-11T17:20:39.834160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518986323771030:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.834254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:39.834306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:39.837834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:39.838233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518985018831253:2069] [node 4] ICS01 socket: 523 reason# EndOfStream
2025-11-11T17:20:39.838306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518985018831177:2069] [node 4] ICP30 unregister session Session# [5:7571518985018831253:2069] VirtualId# [5:7571518985018831248:0]
2025-11-11T17:20:39.838372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518985018831253:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.838529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:39.838583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:39.981911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:39.982174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518982028803720:2062] [node 5] ICS01 socket: 528 reason# EndOfStream
2025-11-11T17:20:39.982226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518982028803647:2062] [node 5] ICP30 unregister session Session# [6:7571518982028803720:2062] VirtualId# [6:7571518982028803712:0]
2025-11-11T17:20:39.982267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518982028803720:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:39.982367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:39.982413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:40.158305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518991813297720:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:40.158365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518991813297721:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:40.178153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518994082056174:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:40.178213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518994082056175:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:40.209952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518992010108309:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:40.209999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518992010108310:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:40.234048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518993902355320:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:40.234115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518993902355321:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:40.234148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518993902355322:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:40.256043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518992719395000:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:40.256104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518992719395001:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:40.256134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518992719395002:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:40.267754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518993701153821:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:40.267797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518993701153822:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:40.267826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518993701153823:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:40.529747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518992010108336:2071] [node 4] ICP01 ready to work
2025-11-11T17:20:40.559530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518992719395016:2060] [node 4] ICP01 ready to work
2025-11-11T17:20:40.628875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518992010108353:2072] [node 5] ICP01 ready to work
2025-11-11T17:20:40.650607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518991813297720:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(211025126ns) {Configure:104(211017204ns) {ProcessConfigured:135(110964ns) {StartInitialHandshake:157(106729ns) {DropHandshakes:469(3322ns) {DropIncomingHandshake:441(682ns) DropOutgoingHandshake:456(433ns)} GenerateSessionVirtualId:988(1238ns) PrepareNewSessionHandshake:476(2266ns) {SwitchToState:178(625ns)}}}}}
2025-11-11T17:20:40.665181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518994082056174:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(227177742ns) {Configure:104(219212688ns) {ProcessConfigured:135(89755ns) {StartInitialHandshake:157(83670ns) {DropHandshakes:469(2759ns) {DropIncomingHandshake:441(747ns) DropOutgoingHandshake:456(474ns)} GenerateSessionVirtualId:988(1365ns) PrepareNewSessionHandshake:476(1807ns) {SwitchToState:178(585ns)}}}}}
2025-11-11T17:20:40.665706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518994082056239:2075] [node 5] ICP01 ready to work
2025-11-11T17:20:40.666061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518994082056240:2079] [node 4] ICP01 ready to work
2025-11-11T17:20:40.666324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518994082056241:2080] [node 6] ICP01 ready to work
2025-11-11T17:20:40.651264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518991813297799:2103] [node 4] ICP01 ready to work
2025-11-11T17:20:40.676548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518992719395016:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(85016879ns) {Configure:104(85009295ns) {ProcessConfigured:135(319441ns) {StartInitialHandshake:157(316062ns) {DropHandshakes:469(3938ns) {DropIncomingHandshake:441(1118ns) DropOutgoingHandshake:456(1028ns)} GenerateSessionVirtualId:988(1228ns) PrepareNewSessionHandshake:476(3800ns) {SwitchToState:178(1713ns)}}}}}
2025-11-11T17:20:40.677063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518992719395066:2065] [node 6] ICP01 ready to work
2025-11-11T17:20:40.723339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518992719395002:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(45914529ns) {Configure:104(45908662ns) {ProcessConfigured:135(85197ns) {StartInitialHandshake:157(81840ns) {DropHandshakes:469(2757ns) {DropIncomingHandshake:441(526ns) DropOutgoingHandshake:456(401ns)} GenerateSessionVirtualId:988(896ns) PrepareNewSessionHandshake:476(1554ns) {SwitchToState:178(642ns)}}}}}
2025-11-11T17:20:40.651627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518991813297800:2118] [node 5] ICP01 ready to work
2025-11-11T17:20:40.652013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518991813297801:2119] [node 6] ICP01 ready to work
2025-11-11T17:20:40.753887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518991813297721:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(85863946ns) {Configure:104(85848474ns) {ProcessConfigured:135(6946604ns) {StartInitialHandshake:157(6943345ns) {DropHandshakes:469(1181ns) {DropIncomingHandshake:441(179ns) DropOutgoingHandshake:456(92ns)} GenerateSessionVirtualId:988(361ns) PrepareNewSessionHandshake:476(4797ns) {SwitchToState:178(2276ns)}}}}}
2025-11-11T17:20:40.754162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518992010108336:2071] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(124853665ns) {Configure:104(124848235ns) {ProcessConfigured:135(58004ns) {StartInitialHandshake:157(55167ns) {DropHandshakes:469(3225ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(514ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:476(1543ns) {SwitchToState:178(189ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.751155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518993902355322:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(234685501ns) {Configure:104(234675629ns) {ProcessConfigured:135(75795ns) {StartInitialHandshake:157(72485ns) {DropHandshakes:469(2527ns) {DropIncomingHandshake:441(619ns) DropOutgoingHandshake:456(377ns)} GenerateSessionVirtualId:988(1009ns) PrepareNewSessionHandshake:476(1488ns) {SwitchToState:178(530ns)}}}}}
2025-11-11T17:20:40.751638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518993902355385:2060] [node 6] ICP01 ready to work
2025-11-11T17:20:40.751897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518993902355386:2063] [node 5] ICP01 ready to work
2025-11-11T17:20:40.770156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518992719395069:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.789880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518993902355398:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
2025-11-11T17:20:40.765884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518992010108376:2086] [node 6] ICP01 ready to work
2025-11-11T17:20:40.785990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518993701153822:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(347560748ns) {Configure:104(347556288ns) {ProcessConfigured:135(11407147ns) {StartInitialHandshake:157(11404310ns) {DropHandshakes:469(2373ns) {DropIncomingHandshake:441(491ns) DropOutgoingHandshake:456(393ns)} GenerateSessionVirtualId:988(748ns) PrepareNewSessionHandshake:476(6890ns) {SwitchToState:178(2750ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.833556Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056243:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.807640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518992010108309:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(29652804ns) {Configure:104(29645073ns) {ProcessConfigured:135(83854ns) {StartInitialHandshake:157(79832ns) {DropHandshakes:469(2707ns) {DropIncomingHandshake:441(567ns) DropOutgoingHandshake:456(479ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:476(1627ns) {SwitchToState:178(877ns)}}}}}
2025-11-11T17:20:40.786464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518993701153886:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.786767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518993701153887:2064] [node 5] ICP01 ready to work
2025-11-11T17:20:40.833613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518994082056243:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:40.834247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518994082056254:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10844
2025-11-11T17:20:40.853989Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056254:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.856630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518991813297799:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(102504689ns) {Configure:104(102499667ns) {ProcessConfigured:135(18822038ns) {StartInitialHandshake:157(18817641ns) {DropHandshakes:469(9607ns) {DropIncomingHandshake:441(543ns) DropOutgoingHandshake:456(730ns)} GenerateSessionVirtualId:988(1462ns) PrepareNewSessionHandshake:476(3694ns) {SwitchToState:178(2039ns)}}}}}
2025-11-11T17:20:40.854163Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518994082056243:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:40.854676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518994082056239:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:40.854727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:40.854779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518994082056239:2075] [node 5] ICP22 created new session: [2:7571518994082056256:2075]
2025-11-11T17:20:40.854827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518994082056256:2075] [node 5] ICS09 handshake done sender: [2:7571518994082056243:2112] self: [2:7571518994082056242:0] peer: [5:7571518992719395080:0] socket: 487 qp: -1
2025-11-11T17:20:40.854864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518994082056256:2075] [node 5] ICS10 traffic start
2025-11-11T17:20:40.854902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:40.855473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518994082056239:2075] [node 5] ICP18 handshake (actor [2:7571518994082056254:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.857998Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518993902355398:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.810074Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518992719395069:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.846046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518992010108381:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55538
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.828861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518992719395079:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
2025-11-11T17:20:40.858062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518992010108382:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55538
2025-11-11T17:20:40.858233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518992010108383:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55538
2025-11-11T17:20:40.860875Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518992010108381:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.866237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518993902355400:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.886461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518993701153888:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:40.887771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518994082056259:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10844
2025-11-11T17:20:40.887836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518994082056260:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10844
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.891120Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518992010108382:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.891388Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518992010108383:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.891609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518992010108310:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:40.829009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518992719395001:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:40.866309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518993902355401:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
2025-11-11T17:20:40.889167Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518993902355401:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.889462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518993902355402:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
2025-11-11T17:20:40.889551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518993902355321:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:40.905804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056247:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.905849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518994082056247:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:40.906451Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056245:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.906469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518994082056245:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:40.856173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518992719395081:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
2025-11-11T17:20:40.875118Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518992719395079:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.875359Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:40.897685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518993701153888:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.930042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518992719395069:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:40.930418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518992719395001:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518992719395069:2112] held: no
2025-11-11T17:20:40.930530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518992719395001:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:40.930578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:40.930629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518992719395001:2049] [node 2] ICP22 created new session: [5:7571518992719395082:2049]
2025-11-11T17:20:40.930676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518992719395082:2049] [node 2] ICS09 handshake done sender: [5:7571518992719395069:2112] self: [5:7571518992719395080:0] peer: [2:7571518994082056242:0] socket: 489 qp: -1
2025-11-11T17:20:40.930715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518992719395082:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:40.930775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:40.931889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518991813297800:2118] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(74984509ns) {Configure:104(74980057ns) {ProcessConfigured:135(19868966ns) {StartInitialHandshake:157(19866549ns) {DropHandshakes:469(2611ns) {DropIncomingHandshake:441(912ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:476(3818ns) {SwitchToState:178(2227ns)}}}}}
2025-11-11T17:20:40.931777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518993902355388:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.931816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518993902355388:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:40.909828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518993701153886:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:40.917772Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518994082056247:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:40.932875Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518994082056245:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:40.933346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518994082056240:2079] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:40.933381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:40.933430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518994082056240:2079] [node 4] ICP22 created new session: [2:7571518994082056263:2079]
2025-11-11T17:20:40.933468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518994082056263:2079] [node 4] ICS09 handshake done sender: [2:7571518994082056247:2114] self: [2:7571518994082056246:0] peer: [4:7571518993902355403:0] socket: 505 qp: -1
2025-11-11T17:20:40.933503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518994082056263:2079] [node 4] ICS10 traffic start
2025-11-11T17:20:40.933536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.942461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518993701153901:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:40.940880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518994082056175:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:40.940934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:40.940973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518994082056175:2049] [node 3] ICP22 created new session: [2:7571518994082056265:2049]
2025-11-11T17:20:40.941014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518994082056265:2049] [node 3] ICS09 handshake done sender: [2:7571518994082056245:2113] self: [2:7571518994082056244:0] peer: [3:7571518992010108391:0] socket: 488 qp: -1
2025-11-11T17:20:40.941048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518994082056265:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:40.941085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:40.941563Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056260:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.945434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518991813297810:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58655
2025-11-11T17:20:40.945568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518991813297811:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58655
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.952833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518994082056269:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10844
2025-11-11T17:20:40.958239Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518993701153901:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.958393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518993701153822:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:40.958922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056250:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.958956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571518994082056250:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.960162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518993701153903:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:40.960549Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518994082056250:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:40.961148Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518993701153901:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:40.964089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518991813297810:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.990195Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518993902355390:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:40.990236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571518993902355390:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:40.990813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518993902355404:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.998371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518993902355405:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
2025-11-11T17:20:40.998653Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518993902355401:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:40.999124Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518993902355388:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:40.999432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518993902355321:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:40.999538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:40.999588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518993902355321:2049] [node 2] ICP22 created new session: [4:7571518993902355408:2049]
2025-11-11T17:20:40.999630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518993902355408:2049] [node 2] ICS09 handshake done sender: [4:7571518993902355401:2117] self: [4:7571518993902355403:0] peer: [2:7571518994082056246:0] socket: 507 qp: -1
2025-11-11T17:20:40.999664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518993902355408:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:40.999698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:41.000359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518993902355390:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:40.961531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518993701153822:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:40.982357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518994082056241:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:40.997639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:41.005713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518994082056241:2080] [node 6] ICP22 created new session: [2:7571518994082056271:2080]
2025-11-11T17:20:40.982595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518992719395016:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:40.983083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518992719395084:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
2025-11-11T17:20:41.015649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518992719395084:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.015936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518997014362383:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
2025-11-11T17:20:41.016822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518992719395079:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:41.017004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518992719395016:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.017049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:41.017093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518992719395016:2060] [node 4] ICP22 created new session: [5:7571518997014362385:2060]
2025-11-11T17:20:41.017134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518997014362385:2060] [node 4] ICS09 handshake done sender: [5:7571518992719395079:2116] self: [5:7571518992719395085:0] peer: [4:7571518993902355389:0] socket: 502 qp: -1
2025-11-11T17:20:41.017170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518997014362385:2060] [node 4] ICS10 traffic start
2025-11-11T17:20:41.017234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:40.961684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:40.969902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518993701153822:2049] [node 2] ICP22 created new session: [6:7571518993701153904:2049]
2025-11-11T17:20:40.977776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518993701153904:2049] [node 2] ICS09 handshake done sender: [6:7571518993701153901:2116] self: [6:7571518993701153902:0] peer: [2:7571518994082056249:0] socket: 521 qp: -1
2025-11-11T17:20:40.977839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518993701153904:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:40.977899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:41.006086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571518993701153822:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(44292385ns) {HandleHandshakeStatus:329(44290545ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(4979ns) SwitchToState:178(460ns) ProcessPendingSessionEvents:495(27782227ns) {ForwardSessionEventToSession:612(12959465ns) ForwardSessionEventToSession:612(7837ns) ForwardSessionEventToSession:612(2618ns) ForwardSessionEventToSession:612(6712200ns) ForwardSessionEventToSession:612(3364ns) ForwardSessionEventToSession:612(8054515ns) ForwardSessionEventToSession:612(6901ns) ForwardSessionEventToSession:612(6400ns) ForwardSessionEventToSession:612(5449ns) ForwardSessionEventToSession:612(2370ns)}}}
2025-11-11T17:20:41.008339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518997996121202:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:41.008392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518997996121203:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:41.022537Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518993701153888:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:41.008769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518993902355385:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.008855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:41.034216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518993701153886:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.034311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:41.034356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518993701153886:2061] [node 4] ICP22 created new session: [6:7571518997996121205:2061]
2025-11-11T17:20:41.034403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518997996121205:2061] [node 4] ICS09 handshake done sender: [6:7571518993701153888:2111] self: [6:7571518993701153900:0] peer: [4:7571518993902355387:0] socket: 497 qp: -1
2025-11-11T17:20:41.034438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518997996121205:2061] [node 4] ICS10 traffic start
2025-11-11T17:20:41.034471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.054949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518992010108394:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55538
2025-11-11T17:20:41.008902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518993902355385:2060] [node 6] ICP22 created new session: [4:7571518998197322706:2060]
2025-11-11T17:20:41.008947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518998197322706:2060] [node 6] ICS09 handshake done sender: [4:7571518993902355388:2111] self: [4:7571518993902355387:0] peer: [6:7571518993701153900:0] socket: 495 qp: -1
2025-11-11T17:20:41.008984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518998197322706:2060] [node 6] ICS10 traffic start
2025-11-11T17:20:41.009020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:41.027119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518994082056271:2080] [node 6] ICS09 handshake done sender: [2:7571518994082056250:2115] self: [2:7571518994082056249:0] peer: [6:7571518993701153902:0] socket: 520 qp: -1
2025-11-11T17:20:41.018034Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518993902355402:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.018082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571518993902355386:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.027187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518994082056271:2080] [node 6] ICS10 traffic start
2025-11-11T17:20:41.018140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:41.018183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518993902355386:2063] [node 5] ICP22 created new session: [4:7571518998197322708:2063]
2025-11-11T17:20:41.027230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:41.018220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518998197322708:2063] [node 5] ICS09 handshake done sender: [4:7571518993902355390:2112] self: [4:7571518993902355389:0] peer: [5:7571518992719395085:0] socket: 496 qp: -1
2025-11-11T17:20:41.027708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571518994082056241:2080] [node 6] ICP35 event processing took too much time PendingConnection:278(48893515ns) {HandleHandshakeStatus:329(48890537ns) {DropIncomingHandshake:441(23246ns) DropOutgoingHandshake:456(643ns) SwitchToState:178(721ns) ProcessPendingSessionEvents:495(109812ns) {ForwardSessionEventToSession:612(63204ns) ForwardSessionEventToSession:612(3536ns) ForwardSessionEventToSession:612(3182ns) ForwardSessionEventToSession:612(6006ns) ForwardSessionEventToSession:612(5686ns) ForwardSessionEventToSession:612(3296ns) ForwardSessionEventToSession:612(6048ns) ForwardSessionEventToSession:612(2963ns) ForwardSessionEventToSession:612(5076ns) ForwardSessionEventToSession:612(2783ns)}}}
2025-11-11T17:20:41.038296Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056269:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.038908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518994082056174:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.018250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518998197322708:2063] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.032098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:41.056125Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518993902355405:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.056228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571518993902355385:2060] [node 6] ICP18 handshake (actor [4:7571518993902355402:2118]) is too old
2025-11-11T17:20:41.063279Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571518993902355400:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.063353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518993902355320:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.064764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518993902355322:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.057997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518991813297798:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.058042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518991813297798:2124] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:41.062144Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518991813297811:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.062849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518996108265112:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58655
2025-11-11T17:20:41.062907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518996108265113:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58655
2025-11-11T17:20:41.081795Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518991813297798:2124] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.084399Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518997996121202:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.084544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518993701153887:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.058842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571518994082056259:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.058994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571518994082056175:2049] [node 3] ICP18 handshake (actor [2:7571518994082056259:2117]) is too old
2025-11-11T17:20:41.083407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571518998377023570:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10844
2025-11-11T17:20:41.102921Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518991813297805:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.102961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518991813297805:2126] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:41.103930Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518996108265112:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.103978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518991813297720:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.104033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:41.104077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518991813297720:2048] [node 2] ICP22 created new session: [1:7571518996108265117:2048]
2025-11-11T17:20:41.104124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518996108265117:2048] [node 2] ICS09 handshake done sender: [1:7571518991813297798:2124] self: [1:7571518991813297797:0] peer: [2:7571518998377023569:0] socket: 524 qp: -1
2025-11-11T17:20:41.104156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518996108265117:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:41.104206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:41.090273Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518992010108375:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.090312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518992010108375:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.121317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518998197322713:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.131440Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518993902355405:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:41.109711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518992719395073:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.114433Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518996108265113:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.131655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518993902355320:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.133968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:41.143169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518993902355320:2048] [node 1] ICP22 created new session: [4:7571518998197322714:2048]
2025-11-11T17:20:41.143230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518998197322714:2048] [node 1] ICS09 handshake done sender: [4:7571518993902355405:2120] self: [4:7571518998197322711:0] peer: [1:7571518991813297804:0] socket: 530 qp: -1
2025-11-11T17:20:41.143269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518998197322714:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:41.143323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:41.124644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518997996121209:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:41.109770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571518992719395073:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:41.117331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518997014362388:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
2025-11-11T17:20:41.125092Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518992719395073:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:41.114874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518996108265116:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58655
2025-11-11T17:20:41.129678Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518991813297805:2126] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:41.130262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518991813297799:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.130325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:41.130375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518991813297799:2103] [node 4] ICP22 created new session: [1:7571518996108265123:2103]
2025-11-11T17:20:41.130416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518996108265123:2103] [node 4] ICS09 handshake done sender: [1:7571518991813297805:2126] self: [1:7571518991813297804:0] peer: [4:7571518998197322711:0] socket: 526 qp: -1
2025-11-11T17:20:41.130454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518996108265123:2103] [node 4] ICS10 traffic start
2025-11-11T17:20:41.130487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:41.153696Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.142827Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:41.156865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571518992719395066:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.156934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:41.156982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518992719395066:2065] [node 6] ICP22 created new session: [5:7571518997014362390:2065]
2025-11-11T17:20:41.157031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518997014362390:2065] [node 6] ICS09 handshake done sender: [5:7571518992719395073:2114] self: [5:7571518992719395072:0] peer: [6:7571518997996121208:0] socket: 531 qp: -1
2025-11-11T17:20:41.157067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518997014362390:2065] [node 6] ICS10 traffic start
2025-11-11T17:20:41.157126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:41.181878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518997014362388:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.182068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518992719395000:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.170260Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518996108265116:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.165969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518996305075691:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55538
2025-11-11T17:20:41.166148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518996305075692:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55538
2025-11-11T17:20:41.166262Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518992010108381:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:41.192018Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:41.193514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518992010108310:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518992010108381:2113] held: no
2025-11-11T17:20:41.193794Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518991813297807:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.193832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518991813297807:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:41.182640Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571518994082056269:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:41.193809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518992010108310:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.193864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:41.193911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518992010108310:2049] [node 2] ICP22 created new session: [3:7571518996305075694:2049]
2025-11-11T17:20:41.206415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518996305075694:2049] [node 2] ICS09 handshake done sender: [3:7571518992010108381:2113] self: [3:7571518992010108391:0] peer: [2:7571518994082056244:0] socket: 499 qp: -1
2025-11-11T17:20:41.206465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518996305075694:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:41.210480Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518997996121202:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.210992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518993701153887:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518997996121202:2118] held: no
2025-11-11T17:20:41.211105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518993701153887:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.211167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:41.211214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518993701153887:2064] [node 5] ICP22 created new session: [6:7571518997996121211:2064]
2025-11-11T17:20:41.211281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518997996121211:2064] [node 5] ICS09 handshake done sender: [6:7571518997996121202:2118] self: [6:7571518997996121208:0] peer: [5:7571518992719395072:0] socket: 511 qp: -1
2025-11-11T17:20:41.211321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518997996121211:2064] [node 5] ICS10 traffic start
2025-11-11T17:20:41.211354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:41.211886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518997996121210:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:41.205365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518994082056174:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571518994082056269:2119] held: no
2025-11-11T17:20:41.205503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518994082056174:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.205557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:41.205626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518994082056174:2048] [node 1] ICP22 created new session: [2:7571518998377023571:2048]
2025-11-11T17:20:41.205680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571518998377023571:2048] [node 1] ICS09 handshake done sender: [2:7571518994082056269:2119] self: [2:7571518998377023569:0] peer: [1:7571518991813297797:0] socket: 527 qp: -1
2025-11-11T17:20:41.205719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571518998377023571:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:41.205762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:41.214017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:41.221953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571518992010108310:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(28019110ns) {HandleHandshakeStatus:329(28017019ns) {DropIncomingHandshake:441(1234ns) DropOutgoingHandshake:456(192ns) SwitchToState:178(369ns) ProcessPendingSessionEvents:495(478ns)}}
2025-11-11T17:20:41.222162Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518996305075691:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.222467Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518996305075692:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.223191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518992010108309:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.224007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518997996121213:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:41.225082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518997996121210:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.225207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518993701153821:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.225004Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518992010108375:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.224813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518991813297803:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.224851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518991813297803:2125] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:41.225941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571518998197322716:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62815
2025-11-11T17:20:41.238034Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571518993902355400:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:41.246335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518992010108336:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.246414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:41.246470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518992010108336:2071] [node 4] ICP22 created new session: [3:7571518996305075699:2071]
2025-11-11T17:20:41.246517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518996305075699:2071] [node 4] ICS09 handshake done sender: [3:7571518992010108375:2110] self: [3:7571518992010108374:0] peer: [4:7571518998197322712:0] socket: 498 qp: -1
2025-11-11T17:20:41.246552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518996305075699:2071] [node 4] ICS10 traffic start
2025-11-11T17:20:41.246587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.249839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518993902355322:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.249910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:41.250704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518993902355322:2050] [node 3] ICP22 created new session: [4:7571518998197322717:2050]
2025-11-11T17:20:41.250796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571518998197322717:2050] [node 3] ICS09 handshake done sender: [4:7571518993902355400:2116] self: [4:7571518998197322712:0] peer: [3:7571518992010108374:0] socket: 506 qp: -1
2025-11-11T17:20:41.250835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571518998197322717:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:41.250877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:41.251443Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:41.250525Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518991813297809:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.250561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518991813297809:2128] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.250072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518997014362393:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
2025-11-11T17:20:41.253628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518997014362394:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.254181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518997996121215:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:41.254670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571518993902355322:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:41.255487Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518991813297807:2127] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:41.255896Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518991813297803:2125] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:41.256315Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518991813297809:2128] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:41.256284Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571518997996121213:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.256421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518993701153823:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.256576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518991813297800:2118] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.256633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:41.256674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518991813297800:2118] [node 5] ICP22 created new session: [1:7571518996108265130:2118]
2025-11-11T17:20:41.256724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518996108265130:2118] [node 5] ICS09 handshake done sender: [1:7571518991813297807:2127] self: [1:7571518991813297806:0] peer: [5:7571518997014362392:0] socket: 541 qp: -1
2025-11-11T17:20:41.256764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518996108265130:2118] [node 5] ICS10 traffic start
2025-11-11T17:20:41.256797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.263315Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518997996121210:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:41.262216Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518992010108385:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.262261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518992010108385:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:41.262802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571518996305075702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55538
2025-11-11T17:20:41.263847Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518997014362388:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:41.266534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518993701153821:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.266627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:41.266673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518993701153821:2048] [node 1] ICP22 created new session: [6:7571518997996121217:2048]
2025-11-11T17:20:41.266737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518997996121217:2048] [node 1] ICS09 handshake done sender: [6:7571518997996121210:2121] self: [6:7571518997996121214:0] peer: [1:7571518991813297808:0] socket: 486 qp: -1
2025-11-11T17:20:41.266767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518997996121217:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:41.266796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:41.270906Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:41.269971Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:41.272108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518992719395000:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.272163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:41.272212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518992719395000:2048] [node 1] ICP22 created new session: [5:7571518997014362395:2048]
2025-11-11T17:20:41.272298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518997014362395:2048] [node 1] ICS09 handshake done sender: [5:7571518997014362388:2120] self: [5:7571518997014362392:0] peer: [1:7571518991813297806:0] socket: 542 qp: -1
2025-11-11T17:20:41.272333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518997014362395:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:41.272390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:41.272640Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518992719395000:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:41.273014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571518997014362393:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.273247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518992719395002:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:41.263948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518991813297721:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.264104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:41.264211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518991813297721:2049] [node 3] ICP22 created new session: [1:7571518996108265132:2049]
2025-11-11T17:20:41.264284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518996108265132:2049] [node 3] ICS09 handshake done sender: [1:7571518991813297803:2125] self: [1:7571518991813297802:0] peer: [3:7571518996305075697:0] socket: 525 qp: -1
2025-11-11T17:20:41.264319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518996108265132:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:41.264360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:41.264723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518991813297801:2119] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.264771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:41.264812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518991813297801:2119] [node 6] ICP22 created new session: [1:7571518996108265134:2119]
2025-11-11T17:20:41.264884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518996108265134:2119] [node 6] ICS09 handshake done sender: [1:7571518991813297809:2128] self: [1:7571518991813297808:0] peer: [6:7571518997996121214:0] socket: 515 qp: -1
2025-11-11T17:20:41.264922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518996108265134:2119] [node 6] ICS10 traffic start
2025-11-11T17:20:41.264954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.276192Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518996305075691:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:41.281988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571518997996121219:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2054
2025-11-11T17:20:41.284341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518992010108309:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571518996305075691:2119] held: no
2025-11-11T17:20:41.284460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518992010108309:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.284504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:41.284562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518992010108309:2048] [node 1] ICP22 created new session: [3:7571518996305075704:2048]
2025-11-11T17:20:41.284605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518996305075704:2048] [node 1] ICS09 handshake done sender: [3:7571518996305075691:2119] self: [3:7571518996305075697:0] peer: [1:7571518991813297802:0] socket: 514 qp: -1
2025-11-11T17:20:41.284640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518996305075704:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:41.284679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:41.285092Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571518992010108387:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:41.285116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571518992010108387:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:41.289869Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518992010108385:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:41.286828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571518997014362398:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41538
2025-11-11T17:20:41.290359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571518992010108387:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:41.290642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518992010108376:2086] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.290695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:41.290741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518992010108376:2086] [node 6] ICP22 created new session: [3:7571518996305075707:2086]
2025-11-11T17:20:41.290785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518996305075707:2086] [node 6] ICS09 handshake done sender: [3:7571518992010108385:2116] self: [3:7571518992010108384:0] peer: [6:7571518997996121216:0] socket: 509 qp: -1
2025-11-11T17:20:41.290819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518996305075707:2086] [node 6] ICS10 traffic start
2025-11-11T17:20:41.290854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:41.291187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571518992010108353:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:41.291227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:41.291263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518992010108353:2072] [node 5] ICP22 created new session: [3:7571518996305075709:2072]
2025-11-11T17:20:41.291330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571518996305075709:2072] [node 5] ICS09 handshake done sender: [3:7571518992010108387:2117] self: [3:7571518992010108386:0] peer: [5:7571518997014362397:0] socket: 500 qp: -1
2025-11-11T17:20:41.291355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571518996305075709:2072] [node 5] ICS10 traffic start
2025-11-11T17:20:41.291390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:41.294583Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:41.294307Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:41.298915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518993701153823:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571518997996121213:2122] held: no
2025-11-11T17:20:41.299068Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571518997996121213:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:41.300196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518993701153823:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.300282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:41.300333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518993701153823:2050] [node 3] ICP22 created new session: [6:7571518997996121220:2050]
2025-11-11T17:20:41.300379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571518997996121220:2050] [node 3] ICS09 handshake done sender: [6:7571518997996121213:2122] self: [6:7571518997996121216:0] peer: [3:7571518992010108384:0] socket: 513 qp: -1
2025-11-11T17:20:41.300414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571518997996121220:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:41.300445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:41.304711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571518992719395002:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571518997014362393:2121] held: no
2025-11-11T17:20:41.304873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571518997014362393:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:41.305272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518992719395002:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:41.305352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:41.305407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518992719395002:2050] [node 3] ICP22 created new session: [5:7571518997014362399:2050]
2025-11-11T17:20:41.305455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571518997014362399:2050] [node 3] ICS09 handshake done sender: [5:7571518997014362393:2121] self: [5:7571518997014362397:0] peer: [3:7571518992010108386:0] socket: 546 qp: -1
2025-11-11T17:20:41.305489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571518997014362399:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:41.305525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:41.343351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:41.343593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571518998377023571:2048] [node 1] ICS01 socket: 527 reason# EndOfStream
2025-11-11T17:20:41.343640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518994082056174:2048] [node 1] ICP30 unregister session Session# [2:7571518998377023571:2048] VirtualId# [2:7571518998377023569:0]
2025-11-11T17:20:41.343676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571518998377023571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.343755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:41.343797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:41.349723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:41.349956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518996305075704:2048] [node 1] ICS01 socket: 514 reason# EndOfStream
2025-11-11T17:20:41.349996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518992010108309:2048] [node 1] ICP30 unregister session Session# [3:7571518996305075704:2048] VirtualId# [3:7571518996305075697:0]
2025-11-11T17:20:41.350036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518996305075704:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.350121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:41.350161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:41.353723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:41.353969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518997996121217:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-11-11T17:20:41.354010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518993701153821:2048] [node 1] ICP30 unregister session Session# [6:7571518997996121217:2048] VirtualId# [6:7571518997996121214:0]
2025-11-11T17:20:41.354042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518997996121217:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.354134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:41.354182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:41.354328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:41.353943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:41.354100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518998197322714:2048] [node 1] ICS01 socket: 530 reason# EndOfStream
2025-11-11T17:20:41.354124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518993902355320:2048] [node 1] ICP30 unregister session Session# [4:7571518998197322714:2048] VirtualId# [4:7571518998197322711:0]
2025-11-11T17:20:41.354145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518998197322714:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.354214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:41.354250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:41.354496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518997014362395:2048] [node 1] ICS01 socket: 542 reason# EndOfStream
2025-11-11T17:20:41.354530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518992719395000:2048] [node 1] ICP30 unregister session Session# [5:7571518997014362395:2048] VirtualId# [5:7571518997014362392:0]
2025-11-11T17:20:41.354553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518997014362395:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.354615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:41.354647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:41.384673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:41.384883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571518996305075694:2049] [node 2] ICS01 socket: 499 reason# EndOfStream
2025-11-11T17:20:41.384932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518992010108310:2049] [node 2] ICP30 unregister session Session# [3:7571518996305075694:2049] VirtualId# [3:7571518992010108391:0]
2025-11-11T17:20:41.384972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571518996305075694:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.385051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:41.385087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:41.394309Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:41.394517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518993902355408:2049] [node 2] ICS01 socket: 507 reason# EndOfStream
2025-11-11T17:20:41.394550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518993902355321:2049] [node 2] ICP30 unregister session Session# [4:7571518993902355408:2049] VirtualId# [4:7571518993902355403:0]
2025-11-11T17:20:41.394574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518993902355408:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.394636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:41.394685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:41.393792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:41.394023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518992719395082:2049] [node 2] ICS01 socket: 489 reason# EndOfStream
2025-11-11T17:20:41.394078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518992719395001:2049] [node 2] ICP30 unregister session Session# [5:7571518992719395082:2049] VirtualId# [5:7571518992719395080:0]
2025-11-11T17:20:41.394112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518992719395082:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.394204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:41.394243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:41.396941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:41.397187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518993701153904:2049] [node 2] ICS01 socket: 521 reason# EndOfStream
2025-11-11T17:20:41.397255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518993701153822:2049] [node 2] ICP30 unregister session Session# [6:7571518993701153904:2049] VirtualId# [6:7571518993701153902:0]
2025-11-11T17:20:41.397304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518993701153904:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.397438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:41.397522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:41.459545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:41.459771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571518998197322717:2050] [node 3] ICS01 socket: 506 reason# EndOfStream
2025-11-11T17:20:41.459814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518993902355322:2050] [node 3] ICP30 unregister session Session# [4:7571518998197322717:2050] VirtualId# [4:7571518998197322712:0]
2025-11-11T17:20:41.459852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571518998197322717:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.459961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:41.460012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:41.461063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:41.461234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518997996121220:2050] [node 3] ICS01 socket: 513 reason# EndOfStream
2025-11-11T17:20:41.461270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518993701153823:2050] [node 3] ICP30 unregister session Session# [6:7571518997996121220:2050] VirtualId# [6:7571518997996121216:0]
2025-11-11T17:20:41.461292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518997996121220:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.460351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:41.460534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518997014362399:2050] [node 3] ICS01 socket: 546 reason# EndOfStream
2025-11-11T17:20:41.460572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518992719395002:2050] [node 3] ICP30 unregister session Session# [5:7571518997014362399:2050] VirtualId# [5:7571518997014362397:0]
2025-11-11T17:20:41.460605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518997014362399:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.460671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:41.460710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:41.461354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:41.461393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:41.497668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:41.497894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571518997996121205:2061] [node 4] ICS07 socket disconnect 497 reason# EndOfStream
2025-11-11T17:20:41.497931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518997996121205:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.498020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:41.498060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571518997996121205:2061] [node 4] ICS15 start handshake
2025-11-11T17:20:41.499731Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24565) Couldn't connect to any resolved address
2025-11-11T17:20:41.498582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:41.499962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518993701153886:2061] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:41.500009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518997996121205:2061] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:41.498753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571518997014362385:2060] [node 4] ICS01 socket: 502 reason# EndOfStream
2025-11-11T17:20:41.500045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518993701153886:2061] [node 4] ICP30 unregister session Session# [6:7571518997996121205:2061] VirtualId# [6:7571518993701153900:0]
2025-11-11T17:20:41.498790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518992719395016:2060] [node 4] ICP30 unregister session Session# [5:7571518997014362385:2060] VirtualId# [5:7571518992719395085:0]
2025-11-11T17:20:41.500081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:41.498822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571518997014362385:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.500113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571518993701153886:2061] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:41.500142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:41.498893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:41.498939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:41.512451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571518993701153886:2061] [node 4] ICP33 wake up from error state
2025-11-11T17:20:41.557713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:41.557970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571518997996121211:2064] [node 5] ICS01 socket: 511 reason# EndOfStream
2025-11-11T17:20:41.558010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518993701153887:2064] [node 5] ICP30 unregister session Session# [6:7571518997996121211:2064] VirtualId# [6:7571518997996121208:0]
2025-11-11T17:20:41.558047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571518997996121211:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:41.558146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:41.558189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:41.662210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518995622186202:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:41.662259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518995622186203:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:41.689971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518997071232570:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:41.690018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571518997071232571:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:41.717922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518997537078404:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:41.717970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571518997537078405:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:41.739638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518996744937815:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:41.739681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518996744937816:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:41.739708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571518996744937817:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:41.763549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518996032519614:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:41.763593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518996032519615:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:41.763650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571518996032519616:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:41.785609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518998803492927:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:41.785652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518998803492928:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:41.785718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571518998803492929:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:42.018381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519003098460279:2101] [node 5] ICP01 ready to work
2025-11-11T17:20:42.082246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519001039905174:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:42.062049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518999917153563:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:42.130009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518999917153564:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:42.194960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571518999917153565:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:42.202235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519001039905175:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:42.271458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518997537078405:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(208525158ns) {Configure:104(208517967ns) {ProcessConfigured:135(87538ns) {StartInitialHandshake:157(82623ns) {DropHandshakes:469(3727ns) {DropIncomingHandshake:441(823ns) DropOutgoingHandshake:456(697ns)} GenerateSessionVirtualId:988(1093ns) PrepareNewSessionHandshake:476(2121ns) {SwitchToState:178(562ns)}}}}}
2025-11-11T17:20:42.259246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518995622186203:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(212759614ns) {Configure:104(212753260ns) {ProcessConfigured:135(74308ns) {StartInitialHandshake:157(71404ns) {DropHandshakes:469(3680ns) {DropIncomingHandshake:441(866ns) DropOutgoingHandshake:456(595ns)} GenerateSessionVirtualId:988(1348ns) PrepareNewSessionHandshake:476(1934ns) {SwitchToState:178(609ns)}}}}}
2025-11-11T17:20:42.272055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519001832045765:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:42.272596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519001832045766:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:42.272878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519001832045767:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:42.296529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518996744937816:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(94020537ns) {Configure:104(94014568ns) {ProcessConfigured:135(46854ns) {StartInitialHandshake:157(45091ns) {DropHandshakes:469(1980ns) {DropIncomingHandshake:441(664ns) DropOutgoingHandshake:456(286ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:476(1755ns) {SwitchToState:178(420ns)}}}}}
2025-11-11T17:20:42.306002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571518997537078404:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(248552791ns) {Configure:104(248544889ns) {ProcessConfigured:135(12611839ns) {StartInitialHandshake:157(12608490ns) {DropHandshakes:469(3354ns) {DropIncomingHandshake:441(1083ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:476(7572ns) {SwitchToState:178(2520ns)}}}}}
2025-11-11T17:20:42.288078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518997071232571:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(228035888ns) {Configure:104(228027455ns) {ProcessConfigured:135(54336233ns) {StartInitialHandshake:157(43613878ns) {DropHandshakes:469(4872ns) {DropIncomingHandshake:441(626ns) DropOutgoingHandshake:456(526ns)} GenerateSessionVirtualId:988(3621ns) PrepareNewSessionHandshake:476(4413ns) {SwitchToState:178(2179ns)}}}}}
2025-11-11T17:20:42.311068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519001832045765:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37702388ns) {Configure:104(37698167ns) {ProcessConfigured:135(43523ns) {StartInitialHandshake:157(40774ns) {DropHandshakes:469(1760ns) {DropIncomingHandshake:441(424ns) DropOutgoingHandshake:456(323ns)} GenerateSessionVirtualId:988(520ns) PrepareNewSessionHandshake:476(1986ns) {SwitchToState:178(771ns)}}}}}
2025-11-11T17:20:42.285874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518995622186202:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21372324ns) {Configure:104(21364080ns) {ProcessConfigured:135(82207ns) {StartInitialHandshake:157(79050ns) {DropHandshakes:469(3464ns) {DropIncomingHandshake:441(958ns) DropOutgoingHandshake:456(739ns)} GenerateSessionVirtualId:988(1200ns) PrepareNewSessionHandshake:476(2379ns) {SwitchToState:178(946ns)}}}}}
2025-11-11T17:20:42.313078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518996032519615:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(207034972ns) {Configure:104(207029320ns) {ProcessConfigured:135(51836ns) {StartInitialHandshake:157(49247ns) {DropHandshakes:469(872ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:476(1314ns) {SwitchToState:178(265ns)}}}}}
2025-11-11T17:20:42.313580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519000327486975:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:42.313864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519000327486976:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:42.319534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519001039905174:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(22714243ns) {Configure:104(22709951ns) {ProcessConfigured:135(45204ns) {StartInitialHandshake:157(42405ns) {DropHandshakes:469(1994ns) {DropIncomingHandshake:441(607ns) DropOutgoingHandshake:456(392ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:476(777ns) {SwitchToState:178(142ns)}}}}}
2025-11-11T17:20:42.312503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571518997071232570:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(257551386ns) {Configure:104(257544842ns) {ProcessConfigured:135(18961755ns) {StartInitialHandshake:157(18957885ns) {DropHandshakes:469(3469ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(1099ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:476(4308ns) {SwitchToState:178(1959ns)}}}}}
2025-11-11T17:20:42.294784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571518999917153564:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32711719ns) {Configure:104(32704839ns) {ProcessConfigured:135(70089ns) {StartInitialHandshake:157(67206ns) {DropHandshakes:469(2625ns) {DropIncomingHandshake:441(734ns) DropOutgoingHandshake:456(164ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:476(1848ns) {SwitchToState:178(411ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.334589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519000327486975:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18015607ns) {Configure:104(18011666ns) {ProcessConfigured:135(60792ns) {StartInitialHandshake:157(54784ns) {DropHandshakes:469(2343ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(268ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:476(1823ns) {SwitchToState:178(612ns)}}}}}
2025-11-11T17:20:42.337273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327486983:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.337338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327486984:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.342792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519001832045774:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52580
2025-11-11T17:20:42.342903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519001832045780:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52580
2025-11-11T17:20:42.342980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519001832045781:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52580
2025-11-11T17:20:42.329184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571518996744937815:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(246655759ns) {Configure:104(246650109ns) {ProcessConfigured:135(16488062ns) {StartInitialHandshake:157(16486053ns) {DropHandshakes:469(1694ns) {DropIncomingHandshake:441(511ns) DropOutgoingHandshake:456(374ns)} GenerateSessionVirtualId:988(352ns) PrepareNewSessionHandshake:476(3255ns) {SwitchToState:178(1347ns)}}}}}
2025-11-11T17:20:42.345996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519003098460279:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(311019210ns) {Configure:104(311011710ns) {ProcessConfigured:135(13986612ns) {StartInitialHandshake:157(13984041ns) {DropHandshakes:469(2766ns) {DropIncomingHandshake:441(743ns) DropOutgoingHandshake:456(96ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:476(3316ns) {SwitchToState:178(1602ns)}}}}}
2025-11-11T17:20:42.346429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519003098460301:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:42.346943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571518998803492927:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(296738326ns) {Configure:104(296731800ns) {ProcessConfigured:135(26661388ns) {StartInitialHandshake:157(26659640ns) {DropHandshakes:469(2350ns) {DropIncomingHandshake:441(397ns) DropOutgoingHandshake:456(287ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:476(2247ns) {SwitchToState:178(266ns)}}}}}
2025-11-11T17:20:42.347395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571518996032519614:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(254448779ns) {Configure:104(254444350ns) {ProcessConfigured:135(48994ns) {StartInitialHandshake:157(45283ns) {DropHandshakes:469(2649ns) {DropIncomingHandshake:441(662ns) DropOutgoingHandshake:456(417ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:476(763ns) {SwitchToState:178(233ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.324098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519001366199933:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.394128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518999917153579:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-11T17:20:42.394285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518999917153580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-11T17:20:42.395268Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153579:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.324479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519001366199934:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.370604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519001039905189:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15938
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.421823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519000327486984:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.343887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519001366199942:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:42.430517Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045781:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.370689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519001039905190:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15938
2025-11-11T17:20:42.441756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519000327486975:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.453753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519000327486983:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.473951Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045774:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.474174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518997537078404:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.474605Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153567:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.474656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518999917153567:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.412786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519001366199945:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5388
2025-11-11T17:20:42.412916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519001039905190:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.487570Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518999917153567:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.490086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519001832045784:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52580
2025-11-11T17:20:42.490793Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519001832045774:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:42.491034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518997537078404:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.491584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518995622186203:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.491663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:42.491711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518995622186203:2049] [node 3] ICP22 created new session: [1:7571518999917153583:2049]
2025-11-11T17:20:42.491777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518999917153583:2049] [node 3] ICS09 handshake done sender: [1:7571518999917153567:2112] self: [1:7571518999917153566:0] peer: [3:7571519001832045783:0] socket: 492 qp: -1
2025-11-11T17:20:42.491814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518999917153583:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:42.491845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:42.465837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518996032519616:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.491114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:42.491162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518997537078404:2048] [node 1] ICP22 created new session: [3:7571519001832045785:2048]
2025-11-11T17:20:42.491229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519001832045785:2048] [node 1] ICS09 handshake done sender: [3:7571519001832045774:2114] self: [3:7571519001832045783:0] peer: [1:7571518999917153566:0] socket: 494 qp: -1
2025-11-11T17:20:42.491275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519001832045785:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:42.491330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:42.493764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045771:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.493813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519001832045771:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.498085Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153580:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.458166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519001039905182:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.421724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519001366199946:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5388
2025-11-11T17:20:42.442168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519001366199947:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5388
2025-11-11T17:20:42.453914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519001366199948:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5388
2025-11-11T17:20:42.499922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199946:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.499993Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199945:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.503567Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199947:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.509838Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199948:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.502538Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045780:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.458199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519001039905182:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.525994Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199936:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.546067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327486992:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.546643Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519000327486983:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:42.546839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518996032519616:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.546895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:42.546956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518996032519616:2050] [node 3] ICP22 created new session: [5:7571519000327486993:2050]
2025-11-11T17:20:42.547011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519000327486993:2050] [node 3] ICS09 handshake done sender: [5:7571519000327486983:2114] self: [5:7571519000327486991:0] peer: [3:7571519001832045770:0] socket: 505 qp: -1
2025-11-11T17:20:42.547047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519000327486993:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:42.547087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:42.502980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571518997537078405:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.464373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519001039905175:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(21521081ns) {IncomingHandshake:261(9776599ns) {DropIncomingHandshake:441(579ns) PrepareNewSessionHandshake:476(2438ns) {SwitchToState:178(1088ns)} GenerateSessionVirtualId:988(1147ns) IssueIncomingHandshakeReply:193(20094ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.562309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460302:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.562543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460303:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.563017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460308:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.563074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460309:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.550225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153571:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.526054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519001366199936:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.509885Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519001832045771:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.579854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460313:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.556026Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.575691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327486995:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.474045Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519001039905189:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.550278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518999917153571:2114] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.621745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327486996:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.529956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199930:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.618122Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153569:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.586944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519003098460301:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(61571695ns) {Configure:104(61567421ns) {ProcessConfigured:135(76005ns) {StartInitialHandshake:157(72952ns) {DropHandshakes:469(4131ns) {DropIncomingHandshake:441(1226ns) DropOutgoingHandshake:456(617ns)} GenerateSessionVirtualId:988(1164ns) PrepareNewSessionHandshake:476(2524ns) {SwitchToState:178(736ns)}}}}}
2025-11-11T17:20:42.477837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518996744937816:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.563470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519001832045766:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.577681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:42.582067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571518997537078405:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519001832045780:2116] held: no
2025-11-11T17:20:42.584004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519001832045766:2108] [node 5] ICP22 created new session: [3:7571519001832045791:2108]
2025-11-11T17:20:42.610585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519001832045791:2108] [node 5] ICS09 handshake done sender: [3:7571519001832045771:2112] self: [3:7571519001832045770:0] peer: [5:7571519000327486991:0] socket: 497 qp: -1
2025-11-11T17:20:42.653725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519001832045791:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:42.653819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.587135Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519003098460303:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.529995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519001366199930:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.618151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518999917153569:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.619107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518999917153587:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-11T17:20:42.619235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518999917153588:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-11T17:20:42.621559Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518999917153569:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:42.627155Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153588:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.627254Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153573:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.627275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518999917153573:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.627797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518999917153565:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.627847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:42.627905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518999917153565:2109] [node 6] ICP22 created new session: [1:7571518999917153590:2109]
2025-11-11T17:20:42.627955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518999917153590:2109] [node 6] ICS09 handshake done sender: [1:7571518999917153569:2113] self: [1:7571518999917153568:0] peer: [6:7571519003098460316:0] socket: 500 qp: -1
2025-11-11T17:20:42.627998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518999917153590:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:42.628060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:42.629577Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153587:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.629733Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518999917153573:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:42.647108Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518999917153571:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:42.686040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518999917153565:2109] [node 6] ICP18 handshake (actor [1:7571518999917153587:2119]) is too old
2025-11-11T17:20:42.686463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518999917153563:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.686565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:42.686616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518999917153563:2107] [node 4] ICP22 created new session: [1:7571518999917153596:2107]
2025-11-11T17:20:42.686670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518999917153596:2107] [node 4] ICS09 handshake done sender: [1:7571518999917153573:2115] self: [1:7571518999917153572:0] peer: [4:7571519001039905202:0] socket: 527 qp: -1
2025-11-11T17:20:42.686711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518999917153596:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:42.686746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:42.687315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518995622186202:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.687369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:42.687418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518995622186202:2048] [node 2] ICP22 created new session: [1:7571518999917153598:2048]
2025-11-11T17:20:42.687460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518999917153598:2048] [node 2] ICS09 handshake done sender: [1:7571518999917153571:2114] self: [1:7571518999917153570:0] peer: [2:7571519001366199951:0] socket: 493 qp: -1
2025-11-11T17:20:42.687506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518999917153598:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:42.687544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:42.677811Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16958) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.530744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571518997071232570:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.587374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518998803492927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.525774Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519001039905182:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:42.712703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519001832045766:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(152977395ns) {HandleHandshakeStatus:329(152973839ns) {DropIncomingHandshake:441(33582ns) DropOutgoingHandshake:456(585ns) SwitchToState:178(628ns) ProcessPendingSessionEvents:495(22446629ns) {ForwardSessionEventToSession:612(22383193ns) ForwardSessionEventToSession:612(8805ns) ForwardSessionEventToSession:612(3358ns) ForwardSessionEventToSession:612(11311ns) ForwardSessionEventToSession:612(2881ns) ForwardSessionEventToSession:612(10633ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(2817ns) ForwardSessionEventToSession:612(5075ns) ForwardSessionEventToSession:612(2301ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.723452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519001832045790:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52580
2025-11-11T17:20:42.738719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519001832045793:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52580
2025-11-11T17:20:42.738842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519001832045794:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52580
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.756584Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153575:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.754357Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045793:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.613756Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519003098460302:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.756624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571518999917153575:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.757435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571518999917153600:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-11T17:20:42.681801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327486997:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.554549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519001039905175:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.765950Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571518999917153575:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:42.768114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571518999917153564:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.768166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:42.768212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571518999917153564:2108] [node 5] ICP22 created new session: [1:7571518999917153602:2108]
2025-11-11T17:20:42.768258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571518999917153602:2108] [node 5] ICS09 handshake done sender: [1:7571518999917153575:2116] self: [1:7571518999917153574:0] peer: [5:7571519000327487005:0] socket: 545 qp: -1
2025-11-11T17:20:42.768293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571518999917153602:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:42.768329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.538005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519001366199930:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:42.754773Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519001832045780:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:42.758238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571518997537078405:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.766071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:42.766137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571518997537078405:2049] [node 2] ICP22 created new session: [3:7571519001832045797:2049]
2025-11-11T17:20:42.766182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519001832045797:2049] [node 2] ICS09 handshake done sender: [3:7571519001832045780:2116] self: [3:7571519001832045789:0] peer: [2:7571519001366199929:0] socket: 510 qp: -1
2025-11-11T17:20:42.766219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519001832045797:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:42.766260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:42.766899Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045794:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.626581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519003098460301:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.782010Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045777:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.782058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519001832045777:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.539272Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519001366199936:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:42.554628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:42.626827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460317:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.634132Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519003098460309:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.638954Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519003098460303:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:42.661897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518998803492927:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.661990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:42.662043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518998803492927:2048] [node 1] ICP22 created new session: [6:7571519003098460320:2048]
2025-11-11T17:20:42.662096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519003098460320:2048] [node 1] ICS09 handshake done sender: [6:7571519003098460303:2123] self: [6:7571519003098460316:0] peer: [1:7571518999917153568:0] socket: 521 qp: -1
2025-11-11T17:20:42.662136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519003098460320:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:42.662171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:42.663073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518998803492928:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.664542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460319:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.665016Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519003098460302:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:42.665366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519003098460301:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.665412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:42.665453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519003098460301:2107] [node 4] ICP22 created new session: [6:7571519003098460324:2107]
2025-11-11T17:20:42.665500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519003098460324:2107] [node 4] ICS09 handshake done sender: [6:7571519003098460302:2122] self: [6:7571519003098460318:0] peer: [4:7571519001039905179:0] socket: 514 qp: -1
2025-11-11T17:20:42.665535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519003098460324:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:42.665570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:42.698503Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519000327486996:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.702768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519003098460301:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(37180395ns) {HandleHandshakeStatus:329(37175964ns) {DropIncomingHandshake:441(565ns) DropOutgoingHandshake:456(5064ns) SwitchToState:178(346ns) ProcessPendingSessionEvents:495(12559440ns) {ForwardSessionEventToSession:612(12499238ns) ForwardSessionEventToSession:612(10670ns) ForwardSessionEventToSession:612(3420ns) ForwardSessionEventToSession:612(2794ns) ForwardSessionEventToSession:612(6224ns) ForwardSessionEventToSession:612(8864ns) ForwardSessionEventToSession:612(5802ns) ForwardSessionEventToSession:612(3022ns) ForwardSessionEventToSession:612(2589ns) ForwardSessionEventToSession:612(6814ns)}}}
2025-11-11T17:20:42.733390Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519003098460313:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.750323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519003098460279:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.750553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460326:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.777795Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519003098460308:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.789850Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571518999917153600:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.790331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571518999917153564:2108] [node 5] ICP18 handshake (actor [1:7571518999917153600:2121]) is too old
2025-11-11T17:20:42.794012Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519001832045777:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:42.698565Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519000327486984:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:42.700311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327486999:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.700457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518996032519615:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.703481Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519000327486997:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.705863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519000327486975:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16958) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519000327486984:2115] held: no
2025-11-11T17:20:42.733247Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519000327486999:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.745874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519000327486975:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(27377581ns) {HandleHandshakeStatus:395(27374633ns) {LogHandshakeFail:472(20918ns) {UpdateErrorStateLog:544(5487ns)} DropOutgoingHandshake:456(1318ns) ProcessPendingSessionEvents:495(15902894ns) {DropSessionEvent:512(12571ns) DropSessionEvent:512(6386ns) DropSessionEvent:512(7958734ns) DropSessionEvent:512(8449ns) DropSessionEvent:512(5840ns) DropSessionEvent:512(5536ns) DropSessionEvent:512(7005ns) DropSessionEvent:512(6686ns) DropSessionEvent:512(7874811ns) DropSessionEvent:512(6676ns)}}}
2025-11-11T17:20:42.745973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519000327486975:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.746024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:42.746076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519000327486975:2108] [node 4] ICP22 created new session: [5:7571519000327487003:2108]
2025-11-11T17:20:42.746150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519000327487003:2108] [node 4] ICS09 handshake done sender: [5:7571519000327486984:2115] self: [5:7571519000327486990:0] peer: [4:7571519001039905181:0] socket: 506 qp: -1
2025-11-11T17:20:42.746198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519000327487003:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:42.746246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:42.746894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571518996032519614:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.769884Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519000327486982:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.769908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519000327486982:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.771152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327487006:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.771206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519000327487007:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52820
2025-11-11T17:20:42.785160Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519000327486982:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:42.554677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519001039905175:2109] [node 5] ICP22 created new session: [4:7571519001039905195:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.807085Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519000327486999:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:42.539504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571518997071232571:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.554725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519001039905195:2109] [node 5] ICS09 handshake done sender: [4:7571519001039905182:2112] self: [4:7571519001039905181:0] peer: [5:7571519000327486990:0] socket: 504 qp: -1
2025-11-11T17:20:42.554764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519001039905195:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:42.554820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:42.573958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519001039905194:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15938
2025-11-11T17:20:42.574270Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:42.598278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518996744937816:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519001039905189:2115] held: no
2025-11-11T17:20:42.598713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519001039905197:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15938
2025-11-11T17:20:42.599342Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519001039905189:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:42.599630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518996744937816:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.599696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:42.599745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518996744937816:2049] [node 2] ICP22 created new session: [4:7571519001039905200:2049]
2025-11-11T17:20:42.599791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519001039905200:2049] [node 2] ICS09 handshake done sender: [4:7571519001039905189:2115] self: [4:7571519001039905192:0] peer: [2:7571519001366199935:0] socket: 515 qp: -1
2025-11-11T17:20:42.599823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519001039905200:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:42.599861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:42.619688Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519001039905194:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.619937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518996744937815:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.629667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519001039905180:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.629690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519001039905180:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.639149Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519001039905180:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:42.639386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519001039905174:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.639433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:42.639501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519001039905174:2108] [node 6] ICP22 created new session: [4:7571519001039905204:2108]
2025-11-11T17:20:42.639543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519001039905204:2108] [node 6] ICS09 handshake done sender: [4:7571519001039905180:2111] self: [4:7571519001039905179:0] peer: [6:7571519003098460318:0] socket: 499 qp: -1
2025-11-11T17:20:42.639581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519001039905204:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:42.639623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:42.647511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519001039905206:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15938
2025-11-11T17:20:42.647950Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519001039905194:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:42.648132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518996744937815:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.648210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:42.648261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518996744937815:2048] [node 1] ICP22 created new session: [4:7571519001039905207:2048]
2025-11-11T17:20:42.648307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519001039905207:2048] [node 1] ICS09 handshake done sender: [4:7571519001039905194:2117] self: [4:7571519001039905202:0] peer: [1:7571518999917153572:0] socket: 528 qp: -1
2025-11-11T17:20:42.648344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519001039905207:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:42.648386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:42.739415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519001039905209:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15938
2025-11-11T17:20:42.767473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519001039905209:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.767649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571518996744937817:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.810273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519001039905211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15938
2025-11-11T17:20:42.816610Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:42.819230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518996032519614:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.819300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:42.819344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518996032519614:2048] [node 1] ICP22 created new session: [5:7571519000327487009:2048]
2025-11-11T17:20:42.819388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519000327487009:2048] [node 1] ICS09 handshake done sender: [5:7571519000327486999:2121] self: [5:7571519000327487005:0] peer: [1:7571518999917153574:0] socket: 498 qp: -1
2025-11-11T17:20:42.819423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519000327487009:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:42.819455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:42.819717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518996032519614:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:42.820130Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519000327486996:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:42.801874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519003098460309:2127] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:42.539576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:42.837785Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519001039905209:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:42.821036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519000327486976:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.821124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:42.821170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519000327486976:2109] [node 6] ICP22 created new session: [5:7571519000327487011:2109]
2025-11-11T17:20:42.821222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519000327487011:2109] [node 6] ICS09 handshake done sender: [5:7571519000327486982:2113] self: [5:7571519000327486981:0] peer: [6:7571519003098460327:0] socket: 538 qp: -1
2025-11-11T17:20:42.821251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519000327487011:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:42.821278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:42.822038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571518996032519615:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.822096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:42.822137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571518996032519615:2049] [node 2] ICP22 created new session: [5:7571519000327487013:2049]
2025-11-11T17:20:42.822183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519000327487013:2049] [node 2] ICS09 handshake done sender: [5:7571519000327486996:2119] self: [5:7571519000327487001:0] peer: [2:7571519001366199943:0] socket: 536 qp: -1
2025-11-11T17:20:42.822209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519000327487013:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:42.822236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:42.823154Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:42.826099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571518996032519615:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:42.829861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571518998803492929:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:42.841729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519001832045765:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.829935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518998803492928:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.830015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:42.830063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518998803492928:2049] [node 2] ICP22 created new session: [6:7571519003098460330:2049]
2025-11-11T17:20:42.830118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519003098460330:2049] [node 2] ICS09 handshake done sender: [6:7571519003098460309:2127] self: [6:7571519003098460322:0] peer: [2:7571519001366199937:0] socket: 517 qp: -1
2025-11-11T17:20:42.830156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519003098460330:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:42.830196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:42.830880Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21710) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:42.841851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:42.866070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519003098460279:2101] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21710) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519003098460313:2128] held: no
2025-11-11T17:20:42.539625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518997071232571:2049] [node 3] ICP22 created new session: [2:7571519001366199955:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.841911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519001832045765:2107] [node 4] ICP22 created new session: [3:7571519001832045801:2107]
2025-11-11T17:20:42.852752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571518996744937817:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.539667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519001366199955:2049] [node 3] ICS09 handshake done sender: [2:7571519001366199930:2110] self: [2:7571519001366199929:0] peer: [3:7571519001832045789:0] socket: 501 qp: -1
2025-11-11T17:20:42.873991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460328:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.841968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519001832045801:2107] [node 4] ICS09 handshake done sender: [3:7571519001832045777:2115] self: [3:7571519001832045776:0] peer: [4:7571519001039905210:0] socket: 502 qp: -1
2025-11-11T17:20:42.842007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519001832045801:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:42.842046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:42.881927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519001832045773:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.881962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519001832045773:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.898103Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519001832045773:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:42.878360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:42.903741Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519003098460313:2128] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:42.910360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519001832045767:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.910448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:42.910499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519001832045767:2109] [node 6] ICP22 created new session: [3:7571519001832045804:2109]
2025-11-11T17:20:42.910551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519001832045804:2109] [node 6] ICS09 handshake done sender: [3:7571519001832045773:2113] self: [3:7571519001832045772:0] peer: [6:7571519003098460329:0] socket: 525 qp: -1
2025-11-11T17:20:42.910587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519001832045804:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:42.910625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:42.539703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519001366199955:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:42.894767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571518996744937817:2050] [node 3] ICP22 created new session: [4:7571519001039905212:2050]
2025-11-11T17:20:42.894830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519001039905212:2050] [node 3] ICS09 handshake done sender: [4:7571519001039905209:2120] self: [4:7571519001039905210:0] peer: [3:7571519001832045776:0] socket: 542 qp: -1
2025-11-11T17:20:42.894876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519001039905212:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:42.894932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:42.895427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571518996744937817:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(42506258ns) {HandleHandshakeStatus:329(42502922ns) {DropIncomingHandshake:441(1186ns) DropOutgoingHandshake:456(5062ns) SwitchToState:178(504ns) ProcessPendingSessionEvents:495(119431ns) {ForwardSessionEventToSession:612(71146ns) ForwardSessionEventToSession:612(3973ns) ForwardSessionEventToSession:612(3626ns) ForwardSessionEventToSession:612(8459ns) ForwardSessionEventToSession:612(2869ns) ForwardSessionEventToSession:612(2419ns) ForwardSessionEventToSession:612(6372ns) ForwardSessionEventToSession:612(2293ns) ForwardSessionEventToSession:612(5276ns) ForwardSessionEventToSession:612(2917ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:42.912867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519003098460279:2101] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.912951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:42.912997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519003098460279:2101] [node 5] ICP22 created new session: [6:7571519003098460333:2101]
2025-11-11T17:20:42.913047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519003098460333:2101] [node 5] ICS09 handshake done sender: [6:7571519003098460313:2128] self: [6:7571519003098460327:0] peer: [5:7571519000327486981:0] socket: 539 qp: -1
2025-11-11T17:20:42.913084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519003098460333:2101] [node 5] ICS10 traffic start
2025-11-11T17:20:42.913123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:42.913498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519003098460332:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29990
2025-11-11T17:20:42.539764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:42.540185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519001366199933:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.540256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:42.540304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519001366199933:2107] [node 4] ICP22 created new session: [2:7571519001366199957:2107]
2025-11-11T17:20:42.540343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519001366199957:2107] [node 4] ICS09 handshake done sender: [2:7571519001366199936:2112] self: [2:7571519001366199935:0] peer: [4:7571519001039905192:0] socket: 503 qp: -1
2025-11-11T17:20:42.540376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519001366199957:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:42.540406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:42.610285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519001366199961:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5388
2025-11-11T17:20:42.654027Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199961:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.721810Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199938:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.721859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519001366199938:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.730801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519001366199944:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:42.730826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519001366199944:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:42.731352Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:42.732647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571518997071232570:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519001366199945:2115] held: no
2025-11-11T17:20:42.732810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519001366199962:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5388
2025-11-11T17:20:42.747762Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519001366199945:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:42.771612Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519001366199944:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:42.771829Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519001366199938:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:42.813954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571518997071232570:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.814052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:42.814102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571518997071232570:2048] [node 1] ICP22 created new session: [2:7571519001366199966:2048]
2025-11-11T17:20:42.814151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519001366199966:2048] [node 1] ICS09 handshake done sender: [2:7571519001366199945:2115] self: [2:7571519001366199951:0] peer: [1:7571518999917153570:0] socket: 513 qp: -1
2025-11-11T17:20:42.814194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519001366199966:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:42.814241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:42.814800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519001366199942:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.814856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:42.814896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519001366199942:2108] [node 5] ICP22 created new session: [2:7571519001366199968:2108]
2025-11-11T17:20:42.814944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519001366199968:2108] [node 5] ICS09 handshake done sender: [2:7571519001366199944:2114] self: [2:7571519001366199943:0] peer: [5:7571519000327487001:0] socket: 533 qp: -1
2025-11-11T17:20:42.814974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519001366199968:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:42.815001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:42.815345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519001366199934:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:42.815431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:42.815476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519001366199934:2109] [node 6] ICP22 created new session: [2:7571519001366199970:2109]
2025-11-11T17:20:42.815527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519001366199970:2109] [node 6] ICS09 handshake done sender: [2:7571519001366199938:2113] self: [2:7571519001366199937:0] peer: [6:7571519003098460322:0] socket: 518 qp: -1
2025-11-11T17:20:42.815552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519001366199970:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:42.815609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:42.921908Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:42.923420Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519003098460308:2126] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:42.923742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571518998803492929:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519003098460308:2126] held: no
2025-11-11T17:20:42.923851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571518998803492929:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:42.923933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:42.923986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571518998803492929:2050] [node 3] ICP22 created new session: [6:7571519003098460335:2050]
2025-11-11T17:20:42.924027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519003098460335:2050] [node 3] ICS09 handshake done sender: [6:7571519003098460308:2126] self: [6:7571519003098460329:0] peer: [3:7571519001832045772:0] socket: 529 qp: -1
2025-11-11T17:20:42.924066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519003098460335:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:42.924099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:42.988243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:42.988471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519001832045785:2048] [node 1] ICS01 socket: 494 reason# EndOfStream
2025-11-11T17:20:42.988512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518997537078404:2048] [node 1] ICP30 unregister session Session# [3:7571519001832045785:2048] VirtualId# [3:7571519001832045783:0]
2025-11-11T17:20:42.988550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519001832045785:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:42.988626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:42.988668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:42.989730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:42.990043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519001366199966:2048] [node 1] ICS01 socket: 513 reason# EndOfStream
2025-11-11T17:20:42.990087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571518997071232570:2048] [node 1] ICP30 unregister session Session# [2:7571519001366199966:2048] VirtualId# [2:7571519001366199951:0]
2025-11-11T17:20:42.990124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519001366199966:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:42.990202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:42.990246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:42.997816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:42.998051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519001039905207:2048] [node 1] ICS01 socket: 528 reason# EndOfStream
2025-11-11T17:20:42.998093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518996744937815:2048] [node 1] ICP30 unregister session Session# [4:7571519001039905207:2048] VirtualId# [4:7571519001039905202:0]
2025-11-11T17:20:42.998125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519001039905207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:42.998240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:42.998291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:42.998413Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:42.998617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519000327487009:2048] [node 1] ICS01 socket: 498 reason# EndOfStream
2025-11-11T17:20:42.998651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518996032519614:2048] [node 1] ICP30 unregister session Session# [5:7571519000327487009:2048] VirtualId# [5:7571519000327487005:0]
2025-11-11T17:20:42.998675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519000327487009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:42.998732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:42.998778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:42.998866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:42.999033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519003098460320:2048] [node 1] ICS01 socket: 521 reason# EndOfStream
2025-11-11T17:20:42.999058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518998803492927:2048] [node 1] ICP30 unregister session Session# [6:7571519003098460320:2048] VirtualId# [6:7571519003098460316:0]
2025-11-11T17:20:42.999079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519003098460320:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:42.999135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:42.999177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:43.066666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:43.066906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519001832045797:2049] [node 2] ICS01 socket: 510 reason# EndOfStream
2025-11-11T17:20:43.066948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571518997537078405:2049] [node 2] ICP30 unregister session Session# [3:7571519001832045797:2049] VirtualId# [3:7571519001832045789:0]
2025-11-11T17:20:43.066986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519001832045797:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.067071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:43.067110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:43.077775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:43.078003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519000327487013:2049] [node 2] ICS01 socket: 536 reason# EndOfStream
2025-11-11T17:20:43.078044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518996032519615:2049] [node 2] ICP30 unregister session Session# [5:7571519000327487013:2049] VirtualId# [5:7571519000327487001:0]
2025-11-11T17:20:43.078076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519000327487013:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.078158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:43.078201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:43.077755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:43.078005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519001039905200:2049] [node 2] ICS01 socket: 515 reason# EndOfStream
2025-11-11T17:20:43.078044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518996744937816:2049] [node 2] ICP30 unregister session Session# [4:7571519001039905200:2049] VirtualId# [4:7571519001039905192:0]
2025-11-11T17:20:43.078074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519001039905200:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.078161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:43.078201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:43.078290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:43.078516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519003098460330:2049] [node 2] ICS01 socket: 517 reason# EndOfStream
2025-11-11T17:20:43.078554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518998803492928:2049] [node 2] ICP30 unregister session Session# [6:7571519003098460330:2049] VirtualId# [6:7571519003098460322:0]
2025-11-11T17:20:43.078580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519003098460330:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.078644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:43.078705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:43.129815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:43.129815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:43.130160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519001039905212:2050] [node 3] ICS07 socket disconnect 542 reason# EndOfStream
2025-11-11T17:20:43.130198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519001039905212:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.130310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:43.130356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519001039905212:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:43.131207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519000327486993:2050] [node 3] ICS01 socket: 505 reason# EndOfStream
2025-11-11T17:20:43.131251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571518996032519616:2050] [node 3] ICP30 unregister session Session# [5:7571519000327486993:2050] VirtualId# [5:7571519000327486991:0]
2025-11-11T17:20:43.131280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519000327486993:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.131342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:43.131386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:43.132314Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25805) Couldn't connect to any resolved address
2025-11-11T17:20:43.132620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571518996744937817:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25805) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:43.132679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519001039905212:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:43.132717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571518996744937817:2050] [node 3] ICP30 unregister session Session# [4:7571519001039905212:2050] VirtualId# [4:7571519001039905210:0]
2025-11-11T17:20:43.132764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:43.132794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571518996744937817:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:43.132809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:43.133702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:43.133912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519003098460335:2050] [node 3] ICS01 socket: 529 reason# EndOfStream
2025-11-11T17:20:43.133949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571518998803492929:2050] [node 3] ICP30 unregister session Session# [6:7571519003098460335:2050] VirtualId# [6:7571519003098460329:0]
2025-11-11T17:20:43.133975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519003098460335:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.134055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:43.134092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:43.142494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571518996744937817:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:20:43.185770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:43.185753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:43.186037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519000327487003:2108] [node 4] ICS01 socket: 506 reason# EndOfStream
2025-11-11T17:20:43.186076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519000327486975:2108] [node 4] ICP30 unregister session Session# [5:7571519000327487003:2108] VirtualId# [5:7571519000327486990:0]
2025-11-11T17:20:43.186110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519000327487003:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.186202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:43.186242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:43.186074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519003098460324:2107] [node 4] ICS07 socket disconnect 514 reason# EndOfStream
2025-11-11T17:20:43.186114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519003098460324:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.186198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:43.186245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519003098460324:2107] [node 4] ICS15 start handshake
2025-11-11T17:20:43.187026Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16958) Couldn't connect to any resolved address
2025-11-11T17:20:43.187283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519003098460301:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16958) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:43.187330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519003098460324:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:43.187369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519003098460301:2107] [node 4] ICP30 unregister session Session# [6:7571519003098460324:2107] VirtualId# [6:7571519003098460318:0]
2025-11-11T17:20:43.187413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:43.187444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519003098460301:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:43.187459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:43.199093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519003098460301:2107] [node 4] ICP33 wake up from error state
2025-11-11T17:20:43.252708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:43.252929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519003098460333:2101] [node 5] ICS01 socket: 539 reason# EndOfStream
2025-11-11T17:20:43.252967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519003098460279:2101] [node 5] ICP30 unregister session Session# [6:7571519003098460333:2101] VirtualId# [6:7571519003098460327:0]
2025-11-11T17:20:43.253001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519003098460333:2101] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:43.253090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:43.253127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:43.378834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519006336952478:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:43.378908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519006336952479:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:43.401881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519005910727812:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:43.401924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519005910727813:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:43.425375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519004128442572:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:43.430592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519004128442571:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:43.470892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519003986838153:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:43.470944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519003986838154:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:43.470971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519003986838155:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:43.494034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519005733715828:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:43.494079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519005733715829:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:43.494105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519005733715830:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:43.517873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519005089419983:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:43.517928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519005089419984:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:43.517957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519005089419985:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:43.914446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519005089419997:2060] [node 5] ICP01 ready to work
2025-11-11T17:20:43.905545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519006336952541:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:43.922670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519006336952542:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:43.934834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519005910727876:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:43.954811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519005733715848:2062] [node 4] ICP01 ready to work
2025-11-11T17:20:43.990272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519003986838201:2085] [node 6] ICP01 ready to work
2025-11-11T17:20:43.954391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519005910727875:2108] [node 5] ICP01 ready to work
2025-11-11T17:20:44.033651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519005910727877:2109] [node 6] ICP01 ready to work
2025-11-11T17:20:44.057561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519010028683186:2075] [node 6] ICP01 ready to work
2025-11-11T17:20:44.033821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519004128442651:2098] [node 4] ICP01 ready to work
2025-11-11T17:20:44.033976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519003986838212:2082] [node 5] ICP01 ready to work
2025-11-11T17:20:44.033649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519006336952543:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:44.075011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519008423409948:2099] [node 5] ICP01 ready to work
2025-11-11T17:20:44.095600Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519004128442572:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(253551807ns) {Configure:104(253545765ns) {ProcessConfigured:135(76295ns) {StartInitialHandshake:157(73141ns) {DropHandshakes:469(4014ns) {DropIncomingHandshake:441(1238ns) DropOutgoingHandshake:456(579ns)} GenerateSessionVirtualId:988(850ns) PrepareNewSessionHandshake:476(3123ns) {SwitchToState:178(1230ns)}}}}}
2025-11-11T17:20:44.100756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519005733715828:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(205276880ns) {Configure:104(205270893ns) {ProcessConfigured:135(64164ns) {StartInitialHandshake:157(60021ns) {DropHandshakes:469(3297ns) {DropIncomingHandshake:441(860ns) DropOutgoingHandshake:456(796ns)} GenerateSessionVirtualId:988(1037ns) PrepareNewSessionHandshake:476(2412ns) {SwitchToState:178(1033ns)}}}}}
2025-11-11T17:20:44.089762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519009384387339:2065] [node 4] ICP01 ready to work
2025-11-11T17:20:44.114654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519003986838155:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(269149388ns) {Configure:104(269143355ns) {ProcessConfigured:135(11292946ns) {StartInitialHandshake:157(11289672ns) {DropHandshakes:469(1281ns) {DropIncomingHandshake:441(339ns) DropOutgoingHandshake:456(103ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:476(5668ns) {SwitchToState:178(3065ns)}}}}}
2025-11-11T17:20:44.116690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519008423409954:2103] [node 6] ICP01 ready to work
2025-11-11T17:20:44.119282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519008423409948:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21753019ns) {Configure:104(21748919ns) {ProcessConfigured:135(71038ns) {StartInitialHandshake:157(67022ns) {DropHandshakes:469(3479ns) {DropIncomingHandshake:441(1233ns) DropOutgoingHandshake:456(646ns)} GenerateSessionVirtualId:988(1231ns) PrepareNewSessionHandshake:476(1959ns) {SwitchToState:178(519ns)}}}}}
2025-11-11T17:20:44.127433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519003986838153:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(112384572ns) {Configure:104(112378661ns) {ProcessConfigured:135(80809ns) {StartInitialHandshake:157(78037ns) {DropHandshakes:469(3113ns) {DropIncomingHandshake:441(796ns) DropOutgoingHandshake:456(502ns)} GenerateSessionVirtualId:988(1018ns) PrepareNewSessionHandshake:476(4500ns) {SwitchToState:178(575ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.132747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519003986838154:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(62338702ns) {Configure:104(62331052ns) {ProcessConfigured:135(65128ns) {StartInitialHandshake:157(58177ns) {DropHandshakes:469(2874ns) {DropIncomingHandshake:441(804ns) DropOutgoingHandshake:456(501ns)} GenerateSessionVirtualId:988(800ns) PrepareNewSessionHandshake:476(1571ns) {SwitchToState:178(625ns)}}}}}
2025-11-11T17:20:44.173878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519003986838212:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31720448ns) {Configure:104(31715396ns) {ProcessConfigured:135(30209216ns) {StartInitialHandshake:157(30206738ns) {DropHandshakes:469(3805ns) {DropIncomingHandshake:441(932ns) DropOutgoingHandshake:456(649ns)} GenerateSessionVirtualId:988(1264ns) PrepareNewSessionHandshake:476(3852ns) {SwitchToState:178(935ns)}}}}}
2025-11-11T17:20:44.145994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519010028683186:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(43486551ns) {Configure:104(43481793ns) {ProcessConfigured:135(7023771ns) {StartInitialHandshake:157(7018135ns) {DropHandshakes:469(3385ns) {DropIncomingHandshake:441(641ns) DropOutgoingHandshake:456(542ns)} GenerateSessionVirtualId:988(929ns) PrepareNewSessionHandshake:476(11060ns) {SwitchToState:178(2804ns)}}}}}
2025-11-11T17:20:44.167441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519006336952542:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(119121633ns) {Configure:104(119108557ns) {ProcessConfigured:135(72278ns) {StartInitialHandshake:157(69528ns) {DropHandshakes:469(1723ns) {DropIncomingHandshake:441(376ns) DropOutgoingHandshake:456(792ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:476(4887ns) {SwitchToState:178(955ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.218592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519008423409963:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31477
2025-11-11T17:20:44.219029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805528:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.219753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519008423409963:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.182701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519006336952478:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(167849031ns) {Configure:104(167843944ns) {ProcessConfigured:135(98568ns) {StartInitialHandshake:157(94903ns) {DropHandshakes:469(3503ns) {DropIncomingHandshake:441(862ns) DropOutgoingHandshake:456(705ns)} GenerateSessionVirtualId:988(1508ns) PrepareNewSessionHandshake:476(2309ns) {SwitchToState:178(560ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.237944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805527:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.238997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519008281805527:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.239130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519003986838155:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.239519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519008423409953:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.239553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519008423409953:2129] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.240739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805530:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.240966Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519008423409953:2129] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.235008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519005910727812:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(159860553ns) {Configure:104(159851945ns) {ProcessConfigured:135(20257506ns) {StartInitialHandshake:157(20254862ns) {DropHandshakes:469(4357ns) {DropIncomingHandshake:441(1030ns) DropOutgoingHandshake:456(642ns)} GenerateSessionVirtualId:988(1289ns) PrepareNewSessionHandshake:476(3821ns) {SwitchToState:178(1902ns)}}}}}
2025-11-11T17:20:44.183423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519010631919846:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-11-11T17:20:44.262049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519005733715830:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(345721175ns) {Configure:104(345716215ns) {ProcessConfigured:135(80355ns) {StartInitialHandshake:157(77233ns) {DropHandshakes:469(3376ns) {DropIncomingHandshake:441(751ns) DropOutgoingHandshake:456(588ns)} GenerateSessionVirtualId:988(2348ns) PrepareNewSessionHandshake:476(1452ns) {SwitchToState:178(320ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.270687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683200:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.270842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683201:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.234204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519006336952479:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(299280477ns) {Configure:104(279424995ns) {ProcessConfigured:135(91015ns) {StartInitialHandshake:157(87458ns) {DropHandshakes:469(3353ns) {DropIncomingHandshake:441(856ns) DropOutgoingHandshake:456(813ns)} GenerateSessionVirtualId:988(1439ns) PrepareNewSessionHandshake:476(2031ns) {SwitchToState:178(587ns)}}}}}
2025-11-11T17:20:44.257683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919846:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.283129Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519010028683201:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.241580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519008281805527:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:44.267715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519005089419985:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(347507746ns) {Configure:104(347502317ns) {ProcessConfigured:135(23822534ns) {StartInitialHandshake:157(23817287ns) {DropHandshakes:469(3302ns) {DropIncomingHandshake:441(697ns) DropOutgoingHandshake:456(547ns)} GenerateSessionVirtualId:988(953ns) PrepareNewSessionHandshake:476(6250ns) {SwitchToState:178(3337ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.254380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519004128442571:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(371095980ns) {Configure:104(371089418ns) {ProcessConfigured:135(39774745ns) {StartInitialHandshake:157(39771780ns) {DropHandshakes:469(1301ns) {DropIncomingHandshake:441(221ns) DropOutgoingHandshake:456(222ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(4839ns) {SwitchToState:178(2996ns)}}}}}
2025-11-11T17:20:44.245720Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519008281805528:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.254556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519010205695180:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64083
2025-11-11T17:20:44.278636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519010631919853:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.298457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519005733715828:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.265831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519004128442651:2098] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.294870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519005910727875:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(237021389ns) {Configure:104(237013602ns) {ProcessConfigured:135(107135ns) {StartInitialHandshake:157(103271ns) {DropHandshakes:469(3812ns) {DropIncomingHandshake:441(814ns) DropOutgoingHandshake:456(730ns)} GenerateSessionVirtualId:988(1123ns) PrepareNewSessionHandshake:476(2131ns) {SwitchToState:178(694ns)}}}}}
2025-11-11T17:20:44.278715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519010631919854:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-11-11T17:20:44.258266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519003986838153:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.317956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519005089419997:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(359221061ns) {Configure:104(359215230ns) {ProcessConfigured:135(35976551ns) {StartInitialHandshake:157(35973209ns) {DropHandshakes:469(3511ns) {DropIncomingHandshake:441(711ns) DropOutgoingHandshake:456(710ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:476(3654ns) {SwitchToState:178(773ns)}}}}}
2025-11-11T17:20:44.265904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.295325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695180:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.268297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519003986838155:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.335644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519010028683200:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.285644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919841:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.285690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519010631919841:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.350008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519005089419984:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(210337355ns) {Configure:104(210330113ns) {ProcessConfigured:135(20059772ns) {StartInitialHandshake:157(20055088ns) {DropHandshakes:469(3572ns) {DropIncomingHandshake:441(790ns) DropOutgoingHandshake:456(508ns)} GenerateSessionVirtualId:988(1284ns) PrepareNewSessionHandshake:476(10473ns) {SwitchToState:178(1919ns)}}}}}
2025-11-11T17:20:44.337814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519005733715830:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.295775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519010205695186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64083
2025-11-11T17:20:44.265957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519004128442651:2098] [node 4] ICP22 created new session: [3:7571519008423409967:2098]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.392847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519005089419983:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(124820566ns) {Configure:104(124814352ns) {ProcessConfigured:135(9767035ns) {StartInitialHandshake:157(9763138ns) {DropHandshakes:469(3451ns) {DropIncomingHandshake:441(863ns) DropOutgoingHandshake:456(736ns)} GenerateSessionVirtualId:988(1166ns) PrepareNewSessionHandshake:476(9075ns) {SwitchToState:178(2818ns)}}}}}
2025-11-11T17:20:44.393038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387358:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.393458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387360:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.268388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.398092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387359:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.295937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519005910727812:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.266010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519008423409967:2098] [node 4] ICS09 handshake done sender: [3:7571519008423409953:2129] self: [3:7571519008423409952:0] peer: [4:7571519008281805529:0] socket: 498 qp: -1
2025-11-11T17:20:44.283649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519003986838155:2050] [node 3] ICP22 created new session: [4:7571519008281805532:2050]
2025-11-11T17:20:44.338761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519005733715848:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(263241139ns) {Configure:104(263234736ns) {ProcessConfigured:135(14980088ns) {StartInitialHandshake:157(14976666ns) {DropHandshakes:469(3003ns) {DropIncomingHandshake:441(792ns) DropOutgoingHandshake:456(384ns)} GenerateSessionVirtualId:988(336ns) PrepareNewSessionHandshake:476(2015ns) {SwitchToState:178(884ns)}}}}}
2025-11-11T17:20:44.266049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519008423409967:2098] [node 4] ICS10 traffic start
2025-11-11T17:20:44.266091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:44.339102Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519008423409956:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.339130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519008423409956:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.340771Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519008423409956:2130] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:44.341059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519008423409948:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.341110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:44.341161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519008423409948:2099] [node 5] ICP22 created new session: [3:7571519008423409971:2099]
2025-11-11T17:20:44.341208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519008423409971:2099] [node 5] ICS09 handshake done sender: [3:7571519008423409956:2130] self: [3:7571519008423409955:0] peer: [5:7571519010028683204:0] socket: 499 qp: -1
2025-11-11T17:20:44.341243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519008423409971:2099] [node 5] ICS10 traffic start
2025-11-11T17:20:44.341280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:44.372899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519008423409973:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31477
2025-11-11T17:20:44.406137Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519008423409973:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.323236Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919845:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.354835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.319521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519005910727813:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(304710814ns) {Configure:104(304703379ns) {ProcessConfigured:135(18304019ns) {StartInitialHandshake:157(9497736ns) {DropHandshakes:469(4099ns) {DropIncomingHandshake:441(637ns) DropOutgoingHandshake:456(674ns)} GenerateSessionVirtualId:988(2439ns) PrepareNewSessionHandshake:476(4141ns) {SwitchToState:178(1528ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.307538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519008281805532:2050] [node 3] ICS09 handshake done sender: [4:7571519008281805527:2115] self: [4:7571519008281805529:0] peer: [3:7571519008423409952:0] socket: 504 qp: -1
2025-11-11T17:20:44.319707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695186:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.342346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519010205695191:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64083
2025-11-11T17:20:44.342401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519010205695192:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64083
2025-11-11T17:20:44.373510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519010205695193:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64083
2025-11-11T17:20:44.398604Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695192:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.398702Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695193:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.417051Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695179:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.417082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519010205695179:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.429673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695177:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.323274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519010631919845:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.409824Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519009384387358:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.307589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519008281805532:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:44.383188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683208:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.333981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519006336952541:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(164639013ns) {Configure:104(164634556ns) {ProcessConfigured:135(12017838ns) {StartInitialHandshake:157(12013400ns) {DropHandshakes:469(4052ns) {DropIncomingHandshake:441(618ns) DropOutgoingHandshake:456(620ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:476(4580ns) {SwitchToState:178(1566ns)}}}}}
2025-11-11T17:20:44.409989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519005089419985:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.449889Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519008423409958:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.429724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519010205695177:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.319109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805534:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.321982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:44.322440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519003986838155:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(63049946ns) {HandleHandshakeStatus:329(63047301ns) {DropIncomingHandshake:441(3017ns) DropOutgoingHandshake:456(16980ns) SwitchToState:178(800ns) ProcessPendingSessionEvents:495(108368ns) {ForwardSessionEventToSession:612(65546ns) ForwardSessionEventToSession:612(3435ns) ForwardSessionEventToSession:612(2331ns) ForwardSessionEventToSession:612(8030ns) ForwardSessionEventToSession:612(3077ns) ForwardSessionEventToSession:612(2564ns) ForwardSessionEventToSession:612(4606ns) ForwardSessionEventToSession:612(2710ns) ForwardSessionEventToSession:612(5104ns) ForwardSessionEventToSession:612(2229ns)}}}
2025-11-11T17:20:44.383284Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519008281805534:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.417314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805536:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.421082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519003986838154:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.453796Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519008281805528:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.449939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519008423409958:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.334182Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919853:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.514190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519003986838153:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.514259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:44.514313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519003986838153:2048] [node 1] ICP22 created new session: [4:7571519008281805540:2048]
2025-11-11T17:20:44.514367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519008281805540:2048] [node 1] ICS09 handshake done sender: [4:7571519008281805528:2116] self: [4:7571519008281805531:0] peer: [1:7571519010631919840:0] socket: 510 qp: -1
2025-11-11T17:20:44.514400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519008281805540:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:44.514432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:44.514975Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.383222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683207:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.521403Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.416407Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519009384387360:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.434524Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695191:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.493881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519008423409958:2131] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:44.334431Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919843:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.410607Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519010028683201:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:44.413116Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519010028683200:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:44.426362Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519010028683208:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.426416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519005733715830:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.426509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:44.426565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519005733715830:2050] [node 3] ICP22 created new session: [5:7571519010028683209:2050]
2025-11-11T17:20:44.426611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519010028683209:2050] [node 3] ICS09 handshake done sender: [5:7571519010028683200:2114] self: [5:7571519010028683204:0] peer: [3:7571519008423409955:0] socket: 514 qp: -1
2025-11-11T17:20:44.426656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519010028683209:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:44.426691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:44.427260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519005733715828:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.427310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:44.427357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519005733715828:2048] [node 1] ICP22 created new session: [5:7571519010028683211:2048]
2025-11-11T17:20:44.427398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519010028683211:2048] [node 1] ICS09 handshake done sender: [5:7571519010028683201:2115] self: [5:7571519010028683202:0] peer: [1:7571519010631919842:0] socket: 519 qp: -1
2025-11-11T17:20:44.427424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519010028683211:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:44.427458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:44.442610Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519010028683194:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.442639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519010028683194:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.446914Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519010028683194:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:44.448269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519010028683186:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.448311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:44.448353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519010028683186:2075] [node 6] ICP22 created new session: [5:7571519010028683216:2075]
2025-11-11T17:20:44.449015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519010028683216:2075] [node 6] ICS09 handshake done sender: [5:7571519010028683194:2112] self: [5:7571519010028683193:0] peer: [6:7571519009384387363:0] socket: 517 qp: -1
2025-11-11T17:20:44.449060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519010028683216:2075] [node 6] ICS10 traffic start
2025-11-11T17:20:44.449090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:44.452713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683218:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.479915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519010028683186:2075] [node 6] ICP35 event processing took too much time PendingConnection:278(31384313ns) {HandleHandshakeStatus:329(31381775ns) {DropIncomingHandshake:441(8993ns) DropOutgoingHandshake:456(165ns) SwitchToState:178(126ns) ProcessPendingSessionEvents:495(6873845ns) {ForwardSessionEventToSession:612(6812489ns) ForwardSessionEventToSession:612(13135ns) ForwardSessionEventToSession:612(3641ns) ForwardSessionEventToSession:612(5823ns) ForwardSessionEventToSession:612(2952ns) ForwardSessionEventToSession:612(7370ns) ForwardSessionEventToSession:612(6331ns) ForwardSessionEventToSession:612(2582ns) ForwardSessionEventToSession:612(3223ns) ForwardSessionEventToSession:612(4807ns)}}}
2025-11-11T17:20:44.527358Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519010028683218:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.527507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519005733715829:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.538021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683220:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.538748Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519010028683218:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:44.538978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519005733715829:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.539022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:44.539064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519005733715829:2049] [node 2] ICP22 created new session: [5:7571519010028683222:2049]
2025-11-11T17:20:44.539106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519010028683222:2049] [node 2] ICS09 handshake done sender: [5:7571519010028683218:2120] self: [5:7571519010028683219:0] peer: [2:7571519010205695184:0] socket: 539 qp: -1
2025-11-11T17:20:44.539134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519010028683222:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:44.539165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.435398Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519010205695179:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:44.522720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519003986838153:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.416617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519005089419984:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.334456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519010631919843:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.342598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919854:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.435842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519005910727877:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.355535Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519010631919843:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:44.427314Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519009384387359:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.494704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519008423409976:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31477
2025-11-11T17:20:44.522894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519003986838154:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519008281805534:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.559567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683221:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.435926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:44.523044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805539:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.427470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519005089419997:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.365657Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519010631919841:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:44.510721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519008423409978:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31477
2025-11-11T17:20:44.562236Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519010028683221:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.437555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387364:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.441724Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.509702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519005089419984:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519009384387360:2118] held: no
2025-11-11T17:20:44.520609Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25249) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.533924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519005089419984:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(28841385ns) {HandleHandshakeStatus:395(28835438ns) {LogHandshakeFail:472(16982ns) {UpdateErrorStateLog:544(6685ns)} DropOutgoingHandshake:456(3151ns) ProcessPendingSessionEvents:495(11622417ns) {DropSessionEvent:512(7201576ns) DropSessionEvent:512(6549ns) DropSessionEvent:512(5220ns) DropSessionEvent:512(6217ns) DropSessionEvent:512(4574ns) DropSessionEvent:512(5657ns) DropSessionEvent:512(5232ns) DropSessionEvent:512(5441ns) DropSessionEvent:512(4367462ns) DropSessionEvent:512(5508ns)}}}
2025-11-11T17:20:44.534134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387356:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.534671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519005089419997:2060] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25249) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519009384387359:2117] held: no
2025-11-11T17:20:44.534950Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519009384387360:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:44.538406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387365:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.550876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387366:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.562046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387367:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.562284Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519009384387356:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.562585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519005089419984:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.562637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:44.562685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519005089419984:2049] [node 2] ICP22 created new session: [6:7571519009384387370:2049]
2025-11-11T17:20:44.562742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519009384387370:2049] [node 2] ICS09 handshake done sender: [6:7571519009384387360:2118] self: [6:7571519009384387362:0] peer: [2:7571519010205695178:0] socket: 546 qp: -1
2025-11-11T17:20:44.562782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519009384387370:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:44.562826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:44.578060Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519009384387367:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.578115Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519009384387358:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:44.583396Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519009384387359:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:44.602101Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.607781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519009384387339:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.607829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519005089419985:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.607901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:44.607941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519005089419985:2050] [node 3] ICP22 created new session: [6:7571519009384387373:2050]
2025-11-11T17:20:44.607982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519009384387373:2050] [node 3] ICS09 handshake done sender: [6:7571519009384387358:2116] self: [6:7571519009384387361:0] peer: [3:7571519008423409957:0] socket: 544 qp: -1
2025-11-11T17:20:44.608011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519009384387373:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:44.608038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:44.608220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519005089419985:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:44.608355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519005089419997:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.608385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:44.608442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519005089419997:2060] [node 5] ICP22 created new session: [6:7571519009384387375:2060]
2025-11-11T17:20:44.608481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519009384387375:2060] [node 5] ICS09 handshake done sender: [6:7571519009384387359:2117] self: [6:7571519009384387363:0] peer: [5:7571519010028683193:0] socket: 545 qp: -1
2025-11-11T17:20:44.608506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519009384387375:2060] [node 5] ICS10 traffic start
2025-11-11T17:20:44.608571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:44.620038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519005089419983:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.535409Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519008281805534:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:44.365775Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519010631919845:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:44.521334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519008423409976:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.435982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519005910727877:2109] [node 6] ICP22 created new session: [2:7571519010205695199:2109]
2025-11-11T17:20:44.566200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519005733715848:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.635222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387378:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.535964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519003986838154:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.525710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519008423409978:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.444223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519006336952478:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.662028Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519009384387356:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.525899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519004128442571:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.530134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519004128442572:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:44.542508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519008423409981:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31477
2025-11-11T17:20:44.563461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519008423409954:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.563540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:44.566960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519008423409954:2103] [node 6] ICP22 created new session: [3:7571519008423409982:2103]
2025-11-11T17:20:44.598035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519008423409982:2103] [node 6] ICS09 handshake done sender: [3:7571519008423409958:2131] self: [3:7571519008423409957:0] peer: [6:7571519009384387361:0] socket: 503 qp: -1
2025-11-11T17:20:44.599318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519008423409982:2103] [node 6] ICS10 traffic start
2025-11-11T17:20:44.605852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:44.614879Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519008423409978:2136] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:44.615262Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.616760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519004128442571:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.616817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:44.616865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519004128442571:2048] [node 1] ICP22 created new session: [3:7571519008423409985:2048]
2025-11-11T17:20:44.616911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519008423409985:2048] [node 1] ICS09 handshake done sender: [3:7571519008423409978:2136] self: [3:7571519008423409979:0] peer: [1:7571519010631919850:0] socket: 534 qp: -1
2025-11-11T17:20:44.616948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519008423409985:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:44.616982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:44.617213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519004128442571:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:44.618111Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.629959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519004128442572:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519008423409976:2135] held: no
2025-11-11T17:20:44.646073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519008423409954:2103] [node 6] ICP35 event processing took too much time PendingConnection:278(85864644ns) {HandleHandshakeStatus:329(85860968ns) {DropIncomingHandshake:441(34132ns) DropOutgoingHandshake:456(536ns) SwitchToState:178(536ns) ProcessPendingSessionEvents:495(10444071ns) {ForwardSessionEventToSession:612(10387373ns) ForwardSessionEventToSession:612(6777ns) ForwardSessionEventToSession:612(8397ns) ForwardSessionEventToSession:612(6067ns) ForwardSessionEventToSession:612(6143ns) ForwardSessionEventToSession:612(3391ns) ForwardSessionEventToSession:612(3113ns) ForwardSessionEventToSession:612(5372ns) ForwardSessionEventToSession:612(2927ns) ForwardSessionEventToSession:612(2923ns)}}}
2025-11-11T17:20:44.669945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519008423409984:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31477
2025-11-11T17:20:44.590502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519010028683225:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41314
2025-11-11T17:20:44.436027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519010205695199:2109] [node 6] ICS09 handshake done sender: [2:7571519010205695179:2112] self: [2:7571519010205695178:0] peer: [6:7571519009384387362:0] socket: 522 qp: -1
2025-11-11T17:20:44.536004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:44.436064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519010205695199:2109] [node 6] ICS10 traffic start
2025-11-11T17:20:44.436100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:44.446777Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519010205695177:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:44.447194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519005910727876:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.447238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:44.447284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519005910727876:2107] [node 4] ICP22 created new session: [2:7571519010205695203:2107]
2025-11-11T17:20:44.447323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519010205695203:2107] [node 4] ICS09 handshake done sender: [2:7571519010205695177:2111] self: [2:7571519010205695176:0] peer: [4:7571519008281805537:0] socket: 505 qp: -1
2025-11-11T17:20:44.447353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519010205695203:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:44.447382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:44.465972Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.467272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519005910727812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519010205695180:2113] held: no
2025-11-11T17:20:44.523238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519010205695194:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64083
2025-11-11T17:20:44.527849Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695185:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.527871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519010205695185:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.528942Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519010205695185:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:44.529206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519005910727875:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.529264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:44.529320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519005910727875:2108] [node 5] ICP22 created new session: [2:7571519010205695207:2108]
2025-11-11T17:20:44.529366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519010205695207:2108] [node 5] ICS09 handshake done sender: [2:7571519010205695185:2114] self: [2:7571519010205695184:0] peer: [5:7571519010028683219:0] socket: 553 qp: -1
2025-11-11T17:20:44.529402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519010205695207:2108] [node 5] ICS10 traffic start
2025-11-11T17:20:44.529438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:44.559243Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519010205695180:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:44.572238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519005910727812:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.572328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:44.572379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519005910727812:2048] [node 1] ICP22 created new session: [2:7571519010205695209:2048]
2025-11-11T17:20:44.572428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519010205695209:2048] [node 1] ICS09 handshake done sender: [2:7571519010205695180:2113] self: [2:7571519010205695187:0] peer: [1:7571519010631919844:0] socket: 513 qp: -1
2025-11-11T17:20:44.572461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519010205695209:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:44.572522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:44.574184Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519010205695189:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.574229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519010205695189:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.586085Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519010205695189:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:44.606325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519005910727813:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.606400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:44.606450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519005910727813:2049] [node 3] ICP22 created new session: [2:7571519010205695212:2049]
2025-11-11T17:20:44.606497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519010205695212:2049] [node 3] ICS09 handshake done sender: [2:7571519010205695189:2116] self: [2:7571519010205695188:0] peer: [3:7571519008423409980:0] socket: 525 qp: -1
2025-11-11T17:20:44.606530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519010205695212:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:44.606567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:44.444316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:44.444366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519006336952478:2048] [node 2] ICP22 created new session: [1:7571519010631919863:2048]
2025-11-11T17:20:44.444417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519010631919863:2048] [node 2] ICS09 handshake done sender: [1:7571519010631919845:2112] self: [1:7571519010631919844:0] peer: [2:7571519010205695187:0] socket: 509 qp: -1
2025-11-11T17:20:44.444453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519010631919863:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:44.444493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:44.444953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519006336952542:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.444997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:44.445042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519006336952542:2107] [node 4] ICP22 created new session: [1:7571519010631919865:2107]
2025-11-11T17:20:44.445078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519010631919865:2107] [node 4] ICS09 handshake done sender: [1:7571519010631919841:2110] self: [1:7571519010631919840:0] peer: [4:7571519008281805531:0] socket: 507 qp: -1
2025-11-11T17:20:44.445108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519010631919865:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:44.445139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:44.445433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519006336952543:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.445478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:44.445514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519006336952543:2109] [node 5] ICP22 created new session: [1:7571519010631919867:2109]
2025-11-11T17:20:44.445546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519010631919867:2109] [node 5] ICS09 handshake done sender: [1:7571519010631919843:2111] self: [1:7571519010631919842:0] peer: [5:7571519010028683202:0] socket: 508 qp: -1
2025-11-11T17:20:44.445577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519010631919867:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:44.445628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:44.495290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519010631919862:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-11-11T17:20:44.522154Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919862:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.523473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519006336952543:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(29337832ns) {HandleHandshakeStatus:329(29335310ns) {DropIncomingHandshake:441(8709ns) DropOutgoingHandshake:456(133ns) SwitchToState:178(131ns) ProcessPendingSessionEvents:495(29065130ns) {ForwardSessionEventToSession:612(4109552ns) ForwardSessionEventToSession:612(5362ns) ForwardSessionEventToSession:612(4019009ns) ForwardSessionEventToSession:612(5464ns) ForwardSessionEventToSession:612(3528ns) ForwardSessionEventToSession:612(16612500ns) ForwardSessionEventToSession:612(10431ns) ForwardSessionEventToSession:612(3733ns) ForwardSessionEventToSession:612(4267462ns) ForwardSessionEventToSession:612(3821ns)}}}
2025-11-11T17:20:44.526282Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919851:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.526306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519010631919851:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.546051Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519010631919851:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:44.581964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519006336952479:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.582084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:44.582132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519006336952479:2049] [node 3] ICP22 created new session: [1:7571519010631919874:2049]
2025-11-11T17:20:44.582181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519010631919874:2049] [node 3] ICS09 handshake done sender: [1:7571519010631919851:2114] self: [1:7571519010631919850:0] peer: [3:7571519008423409979:0] socket: 506 qp: -1
2025-11-11T17:20:44.582219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519010631919874:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:44.582258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:44.598750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519010631919873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-11-11T17:20:44.651429Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919856:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.651463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519010631919856:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.657841Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519010631919873:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.667604Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519010631919856:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:44.667867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519006336952541:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.667934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:44.667984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519006336952541:2108] [node 6] ICP22 created new session: [1:7571519010631919877:2108]
2025-11-11T17:20:44.668032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519010631919877:2108] [node 6] ICS09 handshake done sender: [1:7571519010631919856:2117] self: [1:7571519010631919855:0] peer: [6:7571519009384387377:0] socket: 518 qp: -1
2025-11-11T17:20:44.668064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519010631919877:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:44.668116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:44.671206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519006336952541:2108] [node 6] ICP18 handshake (actor [1:7571519010631919873:2119]) is too old
2025-11-11T17:20:44.625733Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519010028683221:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:44.671500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519005733715848:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.678030Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519008423409976:2135] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:44.536044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519003986838154:2049] [node 2] ICP22 created new session: [4:7571519008281805543:2049]
2025-11-11T17:20:44.671562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:44.671610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519005733715848:2062] [node 4] ICP22 created new session: [5:7571519010028683226:2062]
2025-11-11T17:20:44.671657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519010028683226:2062] [node 4] ICS09 handshake done sender: [5:7571519010028683221:2122] self: [5:7571519010028683224:0] peer: [4:7571519008281805516:0] socket: 524 qp: -1
2025-11-11T17:20:44.671700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519010028683226:2062] [node 4] ICS10 traffic start
2025-11-11T17:20:44.671749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:44.673973Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18753) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:44.691759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519009384387339:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.536083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519008281805543:2049] [node 2] ICS09 handshake done sender: [4:7571519008281805534:2118] self: [4:7571519008281805537:0] peer: [2:7571519010205695176:0] socket: 500 qp: -1
2025-11-11T17:20:44.536113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519008281805543:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:44.536147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:44.569861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805538:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.570203Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519008281805526:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.570230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519008281805526:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.575580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519008281805526:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:44.599387Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519008281805538:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.605929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519003986838212:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.621993Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519008281805520:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.622024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519008281805520:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:44.623668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:44.629670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519003986838212:2082] [node 5] ICP22 created new session: [4:7571519008281805548:2082]
2025-11-11T17:20:44.630426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519008281805548:2082] [node 5] ICS09 handshake done sender: [4:7571519008281805526:2114] self: [4:7571519008281805516:0] peer: [5:7571519010028683224:0] socket: 559 qp: -1
2025-11-11T17:20:44.633269Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519008281805520:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:44.642038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519008281805548:2082] [node 5] ICS10 traffic start
2025-11-11T17:20:44.650127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:44.650610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519003986838212:2082] [node 5] ICP35 event processing took too much time PendingConnection:278(50939237ns) {HandleHandshakeStatus:329(50936948ns) {DropIncomingHandshake:441(1233ns) DropOutgoingHandshake:456(1142ns) SwitchToState:178(1885ns) ProcessPendingSessionEvents:495(105479ns) {ForwardSessionEventToSession:612(55515ns) ForwardSessionEventToSession:612(3047ns) ForwardSessionEventToSession:612(5990ns) ForwardSessionEventToSession:612(5659ns) ForwardSessionEventToSession:612(3444ns) ForwardSessionEventToSession:612(5790ns) ForwardSessionEventToSession:612(3381ns) ForwardSessionEventToSession:612(3121ns) ForwardSessionEventToSession:612(5637ns) ForwardSessionEventToSession:612(2799ns)}}}
2025-11-11T17:20:44.650729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519003986838212:2082] [node 5] ICP18 handshake (actor [4:7571519008281805538:2120]) is too old
2025-11-11T17:20:44.654402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519003986838201:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:44.654450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:44.654505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519003986838201:2085] [node 6] ICP22 created new session: [4:7571519008281805550:2085]
2025-11-11T17:20:44.654552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519008281805550:2085] [node 6] ICS09 handshake done sender: [4:7571519008281805520:2112] self: [4:7571519008281805519:0] peer: [6:7571519009384387372:0] socket: 502 qp: -1
2025-11-11T17:20:44.654593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519008281805550:2085] [node 6] ICS10 traffic start
2025-11-11T17:20:44.654628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:44.670290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519008281805546:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16713
2025-11-11T17:20:44.699990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519008281805546:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:44.707360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519003986838201:2085] [node 6] ICP18 handshake (actor [4:7571519008281805546:2122]) is too old
2025-11-11T17:20:44.698277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519004128442572:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.698344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:44.698400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519004128442572:2049] [node 2] ICP22 created new session: [3:7571519008423409987:2049]
2025-11-11T17:20:44.698448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519008423409987:2049] [node 2] ICS09 handshake done sender: [3:7571519008423409976:2135] self: [3:7571519008423409980:0] peer: [2:7571519010205695188:0] socket: 552 qp: -1
2025-11-11T17:20:44.698483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519008423409987:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:44.698519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:44.691868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:44.691918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519009384387339:2065] [node 4] ICP22 created new session: [6:7571519009384387380:2065]
2025-11-11T17:20:44.691990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519009384387380:2065] [node 4] ICS09 handshake done sender: [6:7571519009384387356:2114] self: [6:7571519009384387372:0] peer: [4:7571519008281805519:0] socket: 529 qp: -1
2025-11-11T17:20:44.692026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519009384387380:2065] [node 4] ICS10 traffic start
2025-11-11T17:20:44.692064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:44.692678Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:44.706450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519005089419983:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519009384387367:2122] held: no
2025-11-11T17:20:44.706815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519009384387379:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25393
2025-11-11T17:20:44.710693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519005733715848:2062] [node 4] ICP35 event processing took too much time PendingConnection:278(35657406ns) {HandleHandshakeStatus:329(35654176ns) {DropIncomingHandshake:441(1535ns) DropOutgoingHandshake:456(5902ns) SwitchToState:178(686ns) ProcessPendingSessionEvents:495(35215179ns) {ForwardSessionEventToSession:612(6391520ns) ForwardSessionEventToSession:612(9551867ns) ForwardSessionEventToSession:612(4225738ns) ForwardSessionEventToSession:612(14562ns) ForwardSessionEventToSession:612(1629551ns) ForwardSessionEventToSession:612(3716ns) ForwardSessionEventToSession:612(3673ns) ForwardSessionEventToSession:612(3902134ns) ForwardSessionEventToSession:612(4157ns) ForwardSessionEventToSession:612(3809380ns)}}}
2025-11-11T17:20:44.711105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519005733715848:2062] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:44.712977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519009384387367:2122] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:44.713285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519005089419983:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:44.713346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:44.713390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519005089419983:2048] [node 1] ICP22 created new session: [6:7571519009384387382:2048]
2025-11-11T17:20:44.713448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519009384387382:2048] [node 1] ICS09 handshake done sender: [6:7571519009384387367:2122] self: [6:7571519009384387377:0] peer: [1:7571519010631919855:0] socket: 555 qp: -1
2025-11-11T17:20:44.713486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519009384387382:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:44.713520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:44.742197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:44.742413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519010205695209:2048] [node 1] ICS01 socket: 513 reason# EndOfStream
2025-11-11T17:20:44.742459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519005910727812:2048] [node 1] ICP30 unregister session Session# [2:7571519010205695209:2048] VirtualId# [2:7571519010205695187:0]
2025-11-11T17:20:44.742497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519010205695209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.742560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:44.742602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:44.744288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:44.744485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519008423409985:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-11-11T17:20:44.744522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519004128442571:2048] [node 1] ICP30 unregister session Session# [3:7571519008423409985:2048] VirtualId# [3:7571519008423409979:0]
2025-11-11T17:20:44.744552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519008423409985:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.744615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:44.744657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:44.746909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:44.747116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519008281805540:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-11-11T17:20:44.747155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519003986838153:2048] [node 1] ICP30 unregister session Session# [4:7571519008281805540:2048] VirtualId# [4:7571519008281805531:0]
2025-11-11T17:20:44.747186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519008281805540:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.747247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:44.747283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:44.748076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:44.748289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519009384387382:2048] [node 1] ICS01 socket: 555 reason# EndOfStream
2025-11-11T17:20:44.748320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519005089419983:2048] [node 1] ICP30 unregister session Session# [6:7571519009384387382:2048] VirtualId# [6:7571519009384387377:0]
2025-11-11T17:20:44.748349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519009384387382:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.748407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:44.748438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:44.753843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:44.754203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519010028683211:2048] [node 1] ICS01 socket: 519 reason# EndOfStream
2025-11-11T17:20:44.754267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519005733715828:2048] [node 1] ICP30 unregister session Session# [5:7571519010028683211:2048] VirtualId# [5:7571519010028683202:0]
2025-11-11T17:20:44.754319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519010028683211:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.754454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:44.754531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:44.815381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:44.815633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519008423409987:2049] [node 2] ICS01 socket: 552 reason# EndOfStream
2025-11-11T17:20:44.815670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519004128442572:2049] [node 2] ICP30 unregister session Session# [3:7571519008423409987:2049] VirtualId# [3:7571519008423409980:0]
2025-11-11T17:20:44.815707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519008423409987:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.815808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:44.815851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:44.821806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:44.822085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519010028683222:2049] [node 2] ICS01 socket: 539 reason# EndOfStream
2025-11-11T17:20:44.822124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519005733715829:2049] [node 2] ICP30 unregister session Session# [5:7571519010028683222:2049] VirtualId# [5:7571519010028683219:0]
2025-11-11T17:20:44.822165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519010028683222:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.822298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:44.822354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:44.831766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:44.832018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519009384387370:2049] [node 2] ICS01 socket: 546 reason# EndOfStream
2025-11-11T17:20:44.833315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:44.833532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519008281805543:2049] [node 2] ICS01 socket: 500 reason# EndOfStream
2025-11-11T17:20:44.833567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519003986838154:2049] [node 2] ICP30 unregister session Session# [4:7571519008281805543:2049] VirtualId# [4:7571519008281805537:0]
2025-11-11T17:20:44.833620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519008281805543:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.833791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:44.833844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:44.832058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519005089419984:2049] [node 2] ICP30 unregister session Session# [6:7571519009384387370:2049] VirtualId# [6:7571519009384387362:0]
2025-11-11T17:20:44.834525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519009384387370:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.838049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:44.844621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:44.933336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:44.933552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519008281805532:2050] [node 3] ICS01 socket: 504 reason# EndOfStream
2025-11-11T17:20:44.933616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519003986838155:2050] [node 3] ICP30 unregister session Session# [4:7571519008281805532:2050] VirtualId# [4:7571519008281805529:0]
2025-11-11T17:20:44.933653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519008281805532:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.933732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:44.933778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:44.937814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:44.952546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519009384387373:2050] [node 3] ICS01 socket: 544 reason# EndOfStream
2025-11-11T17:20:44.952590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519005089419985:2050] [node 3] ICP30 unregister session Session# [6:7571519009384387373:2050] VirtualId# [6:7571519009384387361:0]
2025-11-11T17:20:44.952629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519009384387373:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.952724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:44.952770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:44.940080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:44.946915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519010028683209:2050] [node 3] ICS01 socket: 514 reason# EndOfStream
2025-11-11T17:20:44.953046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519005733715830:2050] [node 3] ICP30 unregister session Session# [5:7571519010028683209:2050] VirtualId# [5:7571519010028683204:0]
2025-11-11T17:20:44.953082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519010028683209:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:44.953138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:44.953176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:45.019287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:45.020061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:45.019549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519010028683226:2062] [node 4] ICS07 socket disconnect 524 reason# EndOfStream
2025-11-11T17:20:45.020246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519009384387380:2065] [node 4] ICS01 socket: 529 reason# EndOfStream
2025-11-11T17:20:45.020288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519009384387339:2065] [node 4] ICP30 unregister session Session# [6:7571519009384387380:2065] VirtualId# [6:7571519009384387372:0]
2025-11-11T17:20:45.020321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519009384387380:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:45.020396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:45.020439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:45.019579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519010028683226:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:45.019727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:45.019772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519010028683226:2062] [node 4] ICS15 start handshake
2025-11-11T17:20:45.020635Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18753) Couldn't connect to any resolved address
2025-11-11T17:20:45.020976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519005733715848:2062] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18753) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:45.021041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519010028683226:2062] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:45.021304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519005733715848:2062] [node 4] ICP30 unregister session Session# [5:7571519010028683226:2062] VirtualId# [5:7571519010028683224:0]
2025-11-11T17:20:45.021354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:45.021406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519005733715848:2062] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:45.021422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:45.031451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519005733715848:2062] [node 4] ICP33 wake up from error state
2025-11-11T17:20:45.113786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:45.114024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519009384387375:2060] [node 5] ICS01 socket: 545 reason# EndOfStream
2025-11-11T17:20:45.114071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519005089419997:2060] [node 5] ICP30 unregister session Session# [6:7571519009384387375:2060] VirtualId# [6:7571519009384387363:0]
2025-11-11T17:20:45.114111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519009384387375:2060] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:45.114226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:45.114274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:45.229662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519015195474378:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:45.229702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519015195474379:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:45.270618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519015624754722:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:45.270674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519015624754723:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:45.322050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519016027322699:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:45.322102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519016027322700:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:45.359228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519014350358903:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:45.359276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519014350358904:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:45.359308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519014350358905:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:45.382056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519015621527958:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:45.382101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519015621527959:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:45.382132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519015621527960:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:45.605650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519015195474412:2072] [node 5] ICP01 ready to work
2025-11-11T17:20:45.466030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519014721355066:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:45.466078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519014721355067:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:45.466106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519014721355068:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:45.607564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519016027322720:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:45.659899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519014721355080:2061] [node 4] ICP01 ready to work
2025-11-11T17:20:45.653724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519015624754747:2072] [node 5] ICP01 ready to work
2025-11-11T17:20:45.738505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519014350358931:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:45.755692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519016027322724:2068] [node 5] ICP01 ready to work
2025-11-11T17:20:45.760234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519014721355089:2065] [node 5] ICP01 ready to work
2025-11-11T17:20:45.750557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519015621527971:2059] [node 4] ICP01 ready to work
2025-11-11T17:20:45.754488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519015195474439:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:45.849852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519015621527982:2065] [node 6] ICP01 ready to work
2025-11-11T17:20:45.785463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519014350358942:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
2025-11-11T17:20:45.793697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519015624754784:2076] [node 6] ICP01 ready to work
2025-11-11T17:20:45.794505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519016027322700:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(64575098ns) {Configure:104(64571351ns) {ProcessConfigured:135(63025ns) {StartInitialHandshake:157(60866ns) {DropHandshakes:469(3124ns) {DropIncomingHandshake:441(746ns) DropOutgoingHandshake:456(382ns)} GenerateSessionVirtualId:988(1015ns) PrepareNewSessionHandshake:476(1222ns) {SwitchToState:178(285ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:45.846059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519014721355089:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(85510621ns) {Configure:104(85506081ns) {ProcessConfigured:135(94901ns) {StartInitialHandshake:157(89719ns) {DropHandshakes:469(3128ns) {DropIncomingHandshake:441(771ns) DropOutgoingHandshake:456(557ns)} GenerateSessionVirtualId:988(3273ns) PrepareNewSessionHandshake:476(3933ns) {SwitchToState:178(2932ns)}}}}}
2025-11-11T17:20:45.824702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519015195474443:2071] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:45.820707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519014350358937:2064] [node 6] ICP01 ready to work
2025-11-11T17:20:45.794938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519016027322734:2075] [node 4] ICP01 ready to work
2025-11-11T17:20:45.882735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519015621527958:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17954857ns) {Configure:104(17949701ns) {ProcessConfigured:135(87490ns) {StartInitialHandshake:157(81114ns) {DropHandshakes:469(3118ns) {DropIncomingHandshake:441(670ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(848ns) PrepareNewSessionHandshake:476(1801ns) {SwitchToState:178(820ns)}}}}}
2025-11-11T17:20:45.971241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519015621528036:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:45.821283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519015624754786:2073] [node 4] ICP01 ready to work
2025-11-11T17:20:45.949192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519015624754789:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61043
2025-11-11T17:20:45.949386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519015624754790:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61043
2025-11-11T17:20:45.908463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519016027322771:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43116
2025-11-11T17:20:45.950743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519014721355080:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(110161188ns) {Configure:104(110155666ns) {ProcessConfigured:135(20403708ns) {StartInitialHandshake:157(20400410ns) {DropHandshakes:469(2480ns) {DropIncomingHandshake:441(770ns) DropOutgoingHandshake:456(389ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:476(5059ns) {SwitchToState:178(2209ns)}}}}}
2025-11-11T17:20:45.838458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519014350358969:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
2025-11-11T17:20:45.995285Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519015624754790:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:45.908531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519016027322772:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43116
2025-11-11T17:20:45.953756Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519014350358969:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.021330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322434:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
2025-11-11T17:20:45.980499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519016027322720:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(372619957ns) {Configure:104(372616046ns) {ProcessConfigured:135(26344519ns) {StartInitialHandshake:157(26341987ns) {DropHandshakes:469(2453ns) {DropIncomingHandshake:441(648ns) DropOutgoingHandshake:456(435ns)} GenerateSessionVirtualId:988(979ns) PrepareNewSessionHandshake:476(6184ns) {SwitchToState:178(2867ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.034551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519015624754722:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(436013437ns) {Configure:104(436002847ns) {ProcessConfigured:135(24471102ns) {StartInitialHandshake:157(20675532ns) {DropHandshakes:469(4146ns) {DropIncomingHandshake:441(696ns) DropOutgoingHandshake:456(511ns)} GenerateSessionVirtualId:988(2780ns) PrepareNewSessionHandshake:476(3005ns) {SwitchToState:178(1603ns)}}}}}
2025-11-11T17:20:46.033979Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519014350358942:2080] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.053826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519015621528036:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.021457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322435:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.037985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519015624754789:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.070099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495333:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:46.086072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519015621527971:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(19847433ns)
2025-11-11T17:20:46.086190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519015621527971:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.089329Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519019916495333:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.105986Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.081681Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519016027322772:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.034126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519014350358979:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
2025-11-11T17:20:46.122692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519015624754723:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(296768192ns) {Configure:104(296759364ns) {ProcessConfigured:135(79272ns) {StartInitialHandshake:157(76693ns) {DropHandshakes:469(2837ns) {DropIncomingHandshake:441(621ns) DropOutgoingHandshake:456(599ns)} GenerateSessionVirtualId:988(899ns) PrepareNewSessionHandshake:476(4526ns) {SwitchToState:178(538ns)}}}}}
2025-11-11T17:20:46.021514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322436:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
2025-11-11T17:20:46.117753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519015195474379:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(348451568ns) {Configure:104(348442078ns) {ProcessConfigured:135(35307171ns) {StartInitialHandshake:157(35303959ns) {DropHandshakes:469(2749ns) {DropIncomingHandshake:441(670ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(1066ns) PrepareNewSessionHandshake:476(9289ns) {SwitchToState:178(4709ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.058182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519014721355066:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(451132930ns) {Configure:104(451127461ns) {ProcessConfigured:135(77081ns) {StartInitialHandshake:157(73414ns) {DropHandshakes:469(2829ns) {DropIncomingHandshake:441(827ns) DropOutgoingHandshake:456(455ns)} GenerateSessionVirtualId:988(882ns) PrepareNewSessionHandshake:476(1837ns) {SwitchToState:178(654ns)}}}}}
2025-11-11T17:20:46.112435Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519015621527984:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.097936Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519014350358944:2081] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.086446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519016027322774:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.112497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519015621527984:2066] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.125977Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519015624754792:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.126030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519015624754792:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.141941Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519015624754794:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.141972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519015624754794:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.144552Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519015624754794:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:46.144866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519015624754784:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.144909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:46.144963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519015624754784:2076] [node 6] ICP22 created new session: [2:7571519019919722101:2076]
2025-11-11T17:20:46.145011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519019919722101:2076] [node 6] ICS09 handshake done sender: [2:7571519015624754794:2114] self: [2:7571519015624754793:0] peer: [6:7571519019016322445:0] socket: 522 qp: -1
2025-11-11T17:20:46.145044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519019919722101:2076] [node 6] ICS10 traffic start
2025-11-11T17:20:46.145097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:46.165843Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519015624754792:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:46.168215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519015624754747:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(414011247ns) {Configure:104(413997694ns) {ProcessConfigured:135(18914207ns) {StartInitialHandshake:157(18910715ns) {DropHandshakes:469(2281ns) {DropIncomingHandshake:441(476ns) DropOutgoingHandshake:456(355ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:476(5778ns) {SwitchToState:178(2027ns)}}}}}
2025-11-11T17:20:46.177801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519015621527971:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519015621528036:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.058353Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519019016322436:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.148032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519015195474378:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(438078390ns) {Configure:104(438069919ns) {ProcessConfigured:135(19068504ns) {StartInitialHandshake:157(19064935ns) {DropHandshakes:469(2645ns) {DropIncomingHandshake:441(680ns) DropOutgoingHandshake:456(423ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:476(7846ns) {SwitchToState:178(1978ns)}}}}}
2025-11-11T17:20:46.086487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519016027322774:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.097966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519014350358944:2081] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.203228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519015624754786:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.203330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.206366Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519015621527984:2066] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:46.207213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.148557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519015195474444:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31046
2025-11-11T17:20:46.059973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519014721355068:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.208741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519015621527982:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.094749Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519016027322771:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.098961Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519014350358979:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.208805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:46.062146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519019016322434:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.134350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519016027322724:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(362484420ns) {Configure:104(362479088ns) {ProcessConfigured:135(78936ns) {StartInitialHandshake:157(75461ns) {DropHandshakes:469(2193ns) {DropIncomingHandshake:441(583ns) DropOutgoingHandshake:456(414ns)} GenerateSessionVirtualId:988(710ns) PrepareNewSessionHandshake:476(1828ns) {SwitchToState:178(634ns)}}}}}
2025-11-11T17:20:46.178912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519015195474412:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(407921075ns) {Configure:104(407913156ns) {ProcessConfigured:135(16822402ns) {StartInitialHandshake:157(16818133ns) {DropHandshakes:469(2543ns) {DropIncomingHandshake:441(904ns) DropOutgoingHandshake:456(454ns)} GenerateSessionVirtualId:988(399ns) PrepareNewSessionHandshake:476(4921ns) {SwitchToState:178(2820ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.212506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519015624754786:2073] [node 4] ICP22 created new session: [2:7571519019919722107:2073]
2025-11-11T17:20:46.099075Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519014350358944:2081] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.179200Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519015195474444:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.082063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519014721355089:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.249825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519019919722107:2073] [node 4] ICS09 handshake done sender: [2:7571519015624754792:2113] self: [2:7571519015624754791:0] peer: [4:7571519018645326282:0] socket: 521 qp: -1
2025-11-11T17:20:46.208851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519015621527982:2065] [node 6] ICP22 created new session: [5:7571519019916495340:2065]
2025-11-11T17:20:46.099366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519014350358931:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.146476Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519016027322768:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.109997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519019016322435:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.179914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519019490441750:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31046
2025-11-11T17:20:46.099429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:46.208904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519019916495340:2065] [node 6] ICS09 handshake done sender: [5:7571519015621527984:2066] self: [5:7571519015621527983:0] peer: [6:7571519019016322442:0] socket: 514 qp: -1
2025-11-11T17:20:46.146509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519016027322768:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.269489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519019919722107:2073] [node 4] ICS10 traffic start
2025-11-11T17:20:46.111030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519014721355067:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(264436115ns) {Configure:104(264430541ns) {ProcessConfigured:135(16318475ns) {StartInitialHandshake:157(16315095ns) {DropHandshakes:469(2363ns) {DropIncomingHandshake:441(377ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(345ns) PrepareNewSessionHandshake:476(4484ns) {SwitchToState:178(2224ns)}}}}}
2025-11-11T17:20:46.194485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519019490441745:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31046
2025-11-11T17:20:46.099490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519014350358931:2067] [node 5] ICP22 created new session: [4:7571519018645326280:2067]
2025-11-11T17:20:46.147544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519020322290078:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43116
2025-11-11T17:20:46.271699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:46.208937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519019916495340:2065] [node 6] ICS10 traffic start
2025-11-11T17:20:46.119510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519014721355067:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.208123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441750:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.147604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519020322290079:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43116
2025-11-11T17:20:46.208980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:46.292040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519019919722108:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61043
2025-11-11T17:20:46.292169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519019919722109:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61043
2025-11-11T17:20:46.345765Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519019919722108:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.353784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519015624754722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.353878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519019919722109:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.382317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519015624754786:2073] [node 4] ICP35 event processing took too much time PendingConnection:278(151389765ns) {HandleHandshakeStatus:329(151384193ns) {DropIncomingHandshake:441(19808ns) DropOutgoingHandshake:456(284ns) SwitchToState:178(474ns) ProcessPendingSessionEvents:495(10730206ns) {ForwardSessionEventToSession:612(10648659ns) ForwardSessionEventToSession:612(11576ns) ForwardSessionEventToSession:612(5209ns) ForwardSessionEventToSession:612(3432ns) ForwardSessionEventToSession:612(22116ns) ForwardSessionEventToSession:612(3490ns) ForwardSessionEventToSession:612(3466ns) ForwardSessionEventToSession:612(5418ns) ForwardSessionEventToSession:612(3880ns) ForwardSessionEventToSession:612(11427ns)}}}
2025-11-11T17:20:46.389901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519019919722098:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.389939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519019919722098:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.413784Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519019919722098:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:46.417732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519019919722106:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.417773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519019919722106:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.418782Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519019919722106:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.265609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495343:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:46.277873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519015621527971:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(81696138ns) {HandleHandshakeStatus:395(68364605ns) {LogHandshakeFail:472(19126ns) {UpdateErrorStateLog:544(5457ns)} DropOutgoingHandshake:456(4020ns) ProcessPendingSessionEvents:495(16446934ns) {DropSessionEvent:512(25245ns) DropSessionEvent:512(4658ns) DropSessionEvent:512(4935ns) DropSessionEvent:512(4474ns) DropSessionEvent:512(5289ns) DropSessionEvent:512(2813885ns) DropSessionEvent:512(13558586ns) DropSessionEvent:512(8135ns) DropSessionEvent:512(4543ns) DropSessionEvent:512(5499ns)}}}
2025-11-11T17:20:46.285658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495335:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:46.310850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495344:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:46.336480Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519019916495338:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.345662Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519019916495344:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.345776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519015621527971:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(19760953ns)
2025-11-11T17:20:46.353674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519019916495343:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.369697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519015621528036:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:46.378172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519015621527958:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.389826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519015621527959:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.390686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519015621527958:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16473452ns) {IncomingHandshake:261(16471342ns) {DropIncomingHandshake:441(348ns) PrepareNewSessionHandshake:476(2888ns) {SwitchToState:178(1160ns)} GenerateSessionVirtualId:988(622ns) IssueIncomingHandshakeReply:193(68205ns) {SwitchToState:178(273ns)}}}
2025-11-11T17:20:46.390883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519015621527971:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.417696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519015621527960:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.419019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495349:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:46.422337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:46.099536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519018645326280:2067] [node 5] ICS09 handshake done sender: [4:7571519014350358944:2081] self: [4:7571519014350358943:0] peer: [5:7571519019916495334:0] socket: 509 qp: -1
2025-11-11T17:20:46.208426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519019490441752:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31046
2025-11-11T17:20:46.143211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322446:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.148079Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519016027322768:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:46.444752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519015621527971:2059] [node 4] ICP22 created new session: [5:7571519019916495348:2059]
2025-11-11T17:20:46.445754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519015621527960:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(16198661ns) {IncomingHandshake:261(16196678ns) {DropIncomingHandshake:441(591ns) PrepareNewSessionHandshake:476(2648ns) {SwitchToState:178(563ns)} GenerateSessionVirtualId:988(431ns) IssueIncomingHandshakeReply:193(4454566ns) {SwitchToState:178(3270ns)}}}
2025-11-11T17:20:46.454037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495350:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:46.457695Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519019916495344:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.471455Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519019916495343:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:46.143277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322447:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.217859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519016027322699:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(408012392ns) {Configure:104(408008631ns) {ProcessConfigured:135(23904100ns) {StartInitialHandshake:157(23899975ns) {DropHandshakes:469(2707ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(601ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:476(4485ns) {SwitchToState:178(1977ns)}}}}}
2025-11-11T17:20:46.230580Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441752:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.454292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519015624754723:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.099569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519018645326280:2067] [node 5] ICS10 traffic start
2025-11-11T17:20:46.483301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519019916495348:2059] [node 4] ICS09 handshake done sender: [5:7571519015621528036:2115] self: [5:7571519019916495334:0] peer: [4:7571519014350358943:0] socket: 518 qp: -1
2025-11-11T17:20:46.158361Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519019016322435:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:46.099618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:46.454379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:46.490065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519019916495351:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17954
2025-11-11T17:20:46.496033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519019916495348:2059] [node 4] ICS10 traffic start
2025-11-11T17:20:46.496095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:46.498103Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.499421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519015621527960:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519019916495343:2119] held: no
2025-11-11T17:20:46.499537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519015621527960:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.499614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:46.499664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519015621527960:2050] [node 3] ICP22 created new session: [5:7571519019916495353:2050]
2025-11-11T17:20:46.499709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519019916495353:2050] [node 3] ICS09 handshake done sender: [5:7571519019916495343:2119] self: [5:7571519019916495347:0] peer: [3:7571519020322290074:0] socket: 561 qp: -1
2025-11-11T17:20:46.499741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519019916495353:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:46.499785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:46.508536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519015621527959:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.278917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519019490441755:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31046
2025-11-11T17:20:46.230407Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519016027322774:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.454431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519015624754723:2049] [node 3] ICP22 created new session: [2:7571519019919722117:2049]
2025-11-11T17:20:46.454474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519019919722117:2049] [node 3] ICS09 handshake done sender: [2:7571519019919722098:2116] self: [2:7571519019919722097:0] peer: [3:7571519020322290090:0] socket: 539 qp: -1
2025-11-11T17:20:46.454510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519019919722117:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:46.454543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:46.486104Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.488951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519015624754722:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519019919722108:2118] held: no
2025-11-11T17:20:46.493780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519015624754747:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.493869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:46.502214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519015624754747:2072] [node 5] ICP22 created new session: [2:7571519019919722119:2072]
2025-11-11T17:20:46.517128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519019919722119:2072] [node 5] ICS09 handshake done sender: [2:7571519019919722106:2117] self: [2:7571519019919722105:0] peer: [5:7571519019916495346:0] socket: 515 qp: -1
2025-11-11T17:20:46.533788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519019919722119:2072] [node 5] ICS10 traffic start
2025-11-11T17:20:46.533847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:46.158592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519014721355067:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.249962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519016027322734:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.258669Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519020322290078:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.258724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519016027322720:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.258786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:46.258838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519016027322720:2067] [node 6] ICP22 created new session: [3:7571519020322290082:2067]
2025-11-11T17:20:46.258879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519020322290082:2067] [node 6] ICS09 handshake done sender: [3:7571519016027322774:2114] self: [3:7571519016027322773:0] peer: [6:7571519019016322441:0] socket: 524 qp: -1
2025-11-11T17:20:46.258915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519020322290082:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:46.258951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:46.271185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519020322290086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43116
2025-11-11T17:20:46.283642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:46.289907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519020322290086:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.293036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519016027322734:2075] [node 4] ICP22 created new session: [3:7571519020322290087:2075]
2025-11-11T17:20:46.311496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519016027322699:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.322690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519020322290087:2075] [node 4] ICS09 handshake done sender: [3:7571519016027322768:2111] self: [3:7571519016027322767:0] peer: [4:7571519018645326287:0] socket: 512 qp: -1
2025-11-11T17:20:46.336422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519020322290087:2075] [node 4] ICS10 traffic start
2025-11-11T17:20:46.346047Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519020322290079:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.353815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:46.354132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519016027322700:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.410281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519016027322734:2075] [node 4] ICP35 event processing took too much time PendingConnection:278(159862473ns) {HandleHandshakeStatus:329(159858604ns) {DropIncomingHandshake:441(739ns) DropOutgoingHandshake:456(475ns) SwitchToState:178(4900ns) ProcessPendingSessionEvents:495(20299497ns) {ForwardSessionEventToSession:612(20231183ns) ForwardSessionEventToSession:612(8046ns) ForwardSessionEventToSession:612(7928ns) ForwardSessionEventToSession:612(12183ns) ForwardSessionEventToSession:612(3422ns) ForwardSessionEventToSession:612(5792ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(2203ns) ForwardSessionEventToSession:612(5777ns) ForwardSessionEventToSession:612(3068ns)}}}
2025-11-11T17:20:46.410369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519016027322734:2075] [node 4] ICP18 handshake (actor [3:7571519020322290078:2117]) is too old
2025-11-11T17:20:46.425915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519020322290075:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.425950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519020322290075:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.430524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519020322290092:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43116
2025-11-11T17:20:46.437782Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519020322290075:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:46.446606Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.469858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519016027322699:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519020322290086:2119] held: no
2025-11-11T17:20:46.469966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519016027322724:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.470055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:46.470107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519016027322724:2068] [node 5] ICP22 created new session: [3:7571519020322290094:2068]
2025-11-11T17:20:46.470157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519020322290094:2068] [node 5] ICS09 handshake done sender: [3:7571519020322290075:2115] self: [3:7571519020322290074:0] peer: [5:7571519019916495347:0] socket: 528 qp: -1
2025-11-11T17:20:46.470191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519020322290094:2068] [node 5] ICS10 traffic start
2025-11-11T17:20:46.470252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:46.473907Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519020322290079:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:46.486907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519016027322700:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.486960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:46.487006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519016027322700:2049] [node 2] ICP22 created new session: [3:7571519020322290096:2049]
2025-11-11T17:20:46.487060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519020322290096:2049] [node 2] ICS09 handshake done sender: [3:7571519020322290079:2118] self: [3:7571519020322290090:0] peer: [2:7571519019919722097:0] socket: 542 qp: -1
2025-11-11T17:20:46.487094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519020322290096:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:46.487124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:46.286167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519015195474443:2071] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(167939181ns) {Configure:104(167930126ns) {ProcessConfigured:135(63109181ns) {StartInitialHandshake:157(63106068ns) {DropHandshakes:469(2395ns) {DropIncomingHandshake:441(331ns) DropOutgoingHandshake:456(1085ns)} GenerateSessionVirtualId:988(645ns) PrepareNewSessionHandshake:476(3321ns) {SwitchToState:178(1389ns)}}}}}
2025-11-11T17:20:46.553918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519020322290091:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43116
2025-11-11T17:20:46.554661Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519020322290086:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:46.554933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519016027322699:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.554985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:46.555039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519016027322699:2048] [node 1] ICP22 created new session: [3:7571519020322290098:2048]
2025-11-11T17:20:46.555081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519020322290098:2048] [node 1] ICS09 handshake done sender: [3:7571519020322290086:2119] self: [3:7571519020322290088:0] peer: [1:7571519019490441741:0] socket: 506 qp: -1
2025-11-11T17:20:46.555114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519020322290098:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:46.555150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:46.103707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519018645326277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.540465Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519019916495338:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:46.158659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:46.105910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519014350358937:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(252121709ns) {Configure:104(252110140ns) {ProcessConfigured:135(19806829ns) {StartInitialHandshake:157(19803514ns) {DropHandshakes:469(1125051ns) {DropIncomingHandshake:441(1111ns) DropOutgoingHandshake:456(1718ns)} GenerateSessionVirtualId:988(1860ns) PrepareNewSessionHandshake:476(3748ns) {SwitchToState:178(1024ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.305868Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441745:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.543067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:46.545404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519015621527971:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(138871327ns) {HandleHandshakeStatus:329(138868717ns) {DropIncomingHandshake:441(809ns) DropOutgoingHandshake:456(467ns) SwitchToState:178(475ns) ProcessPendingSessionEvents:495(1211ns)}}
2025-11-11T17:20:46.547033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519015621527959:2049] [node 2] ICP22 created new session: [5:7571519019916495355:2049]
2025-11-11T17:20:46.558400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519019916495355:2049] [node 2] ICS09 handshake done sender: [5:7571519019916495344:2120] self: [5:7571519019916495346:0] peer: [2:7571519019919722105:0] socket: 562 qp: -1
2025-11-11T17:20:46.569009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519019916495355:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:46.572438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:46.582063Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.158708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519014721355067:2049] [node 2] ICP22 created new session: [6:7571519019016322448:2049]
2025-11-11T17:20:46.158758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519019016322448:2049] [node 2] ICS09 handshake done sender: [6:7571519019016322435:2114] self: [6:7571519019016322445:0] peer: [2:7571519015624754793:0] socket: 526 qp: -1
2025-11-11T17:20:46.158798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519019016322448:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:46.158829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:46.207953Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519019016322434:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:46.219821Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.220856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322451:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
2025-11-11T17:20:46.222758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519014721355089:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.228413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322452:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
2025-11-11T17:20:46.230270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:46.250015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519014721355089:2065] [node 5] ICP22 created new session: [6:7571519019016322453:2065]
2025-11-11T17:20:46.273773Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519019016322452:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.285826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519019016322453:2065] [node 5] ICS09 handshake done sender: [6:7571519019016322434:2113] self: [6:7571519019016322442:0] peer: [5:7571519015621527983:0] socket: 525 qp: -1
2025-11-11T17:20:46.285882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519019016322453:2065] [node 5] ICS10 traffic start
2025-11-11T17:20:46.292846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:46.309890Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.312123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519014721355068:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519019016322436:2115] held: no
2025-11-11T17:20:46.312290Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519019016322436:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:46.312628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519014721355080:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.312663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519014721355068:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.312729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:46.312773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519014721355068:2050] [node 3] ICP22 created new session: [6:7571519019016322456:2050]
2025-11-11T17:20:46.312812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519019016322456:2050] [node 3] ICS09 handshake done sender: [6:7571519019016322436:2115] self: [6:7571519019016322441:0] peer: [3:7571519016027322773:0] socket: 527 qp: -1
2025-11-11T17:20:46.312850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519019016322456:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:46.312880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:46.371625Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.373962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322458:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
2025-11-11T17:20:46.377762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519014721355080:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519019016322452:2121] held: no
2025-11-11T17:20:46.385904Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519019016322452:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:46.386158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519014721355080:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.386202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:46.386251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519014721355080:2061] [node 4] ICP22 created new session: [6:7571519019016322459:2061]
2025-11-11T17:20:46.386298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519019016322459:2061] [node 4] ICS09 handshake done sender: [6:7571519019016322452:2121] self: [6:7571519019016322455:0] peer: [4:7571519018645326278:0] socket: 558 qp: -1
2025-11-11T17:20:46.386338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519019016322459:2061] [node 4] ICS10 traffic start
2025-11-11T17:20:46.386378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:46.429708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519014721355089:2065] [node 5] ICP35 event processing took too much time PendingConnection:278(132261571ns) {HandleHandshakeStatus:329(132257984ns) {DropIncomingHandshake:441(3717ns) DropOutgoingHandshake:456(9360ns) SwitchToState:178(458ns) ProcessPendingSessionEvents:495(9340904ns) {ForwardSessionEventToSession:612(9287486ns) ForwardSessionEventToSession:612(5022ns) ForwardSessionEventToSession:612(7779ns) ForwardSessionEventToSession:612(8995ns) ForwardSessionEventToSession:612(2379ns) ForwardSessionEventToSession:612(5719ns) ForwardSessionEventToSession:612(3095ns) ForwardSessionEventToSession:612(5895ns) ForwardSessionEventToSession:612(3177ns) ForwardSessionEventToSession:612(2662ns)}}}
2025-11-11T17:20:46.436828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519014721355089:2065] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:46.468618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322461:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
2025-11-11T17:20:46.546928Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519019016322461:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.554169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519014721355066:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.603131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519015621527958:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.603224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:46.603276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519015621527958:2048] [node 1] ICP22 created new session: [5:7571519019916495357:2048]
2025-11-11T17:20:46.603322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519019916495357:2048] [node 1] ICS09 handshake done sender: [5:7571519019916495338:2118] self: [5:7571519019916495345:0] peer: [1:7571519019490441748:0] socket: 554 qp: -1
2025-11-11T17:20:46.603365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519019916495357:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:46.566050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519015624754747:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(99977856ns) {HandleHandshakeStatus:329(72055015ns) {DropIncomingHandshake:441(33497ns) DropOutgoingHandshake:456(429ns) SwitchToState:178(581ns) ProcessPendingSessionEvents:495(11927142ns) {ForwardSessionEventToSession:612(11861048ns) ForwardSessionEventToSession:612(17029ns) ForwardSessionEventToSession:612(6245ns) ForwardSessionEventToSession:612(7568ns) ForwardSessionEventToSession:612(2456ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(6340ns) ForwardSessionEventToSession:612(2609ns) ForwardSessionEventToSession:612(2205ns) ForwardSessionEventToSession:612(5102ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:46.603395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:46.109782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519014350358904:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.119265Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519018645326277:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.143681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519014350358905:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(413480652ns) {Configure:104(413476418ns) {ProcessConfigured:135(17493267ns) {StartInitialHandshake:157(17489405ns) {DropHandshakes:469(1079ns) {DropIncomingHandshake:441(157ns) DropOutgoingHandshake:456(391ns)} GenerateSessionVirtualId:988(301ns) PrepareNewSessionHandshake:476(2624ns) {SwitchToState:178(784ns)}}}}}
2025-11-11T17:20:46.143879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519014350358905:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.181661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519014350358937:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(19970525ns) {IncomingHandshake:261(19968724ns) {DropIncomingHandshake:441(1456ns) PrepareNewSessionHandshake:476(4918ns) {SwitchToState:178(4372ns)} GenerateSessionVirtualId:988(361ns) IssueIncomingHandshakeReply:193(12612ns)}}
2025-11-11T17:20:46.201782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519018645326289:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
2025-11-11T17:20:46.226225Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519014350358979:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:46.226477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519014350358904:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.226535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:46.226581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519014350358904:2049] [node 2] ICP22 created new session: [4:7571519018645326291:2049]
2025-11-11T17:20:46.226629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519018645326291:2049] [node 2] ICS09 handshake done sender: [4:7571519014350358979:2115] self: [4:7571519018645326282:0] peer: [2:7571519015624754791:0] socket: 523 qp: -1
2025-11-11T17:20:46.226666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519018645326291:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:46.226700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:46.329794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519018645326290:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
2025-11-11T17:20:46.341781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519018645326283:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.341824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519018645326283:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.362432Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519018645326283:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:46.362561Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519014350358969:2106] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:46.362848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519014350358905:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.362914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:46.362963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519014350358905:2050] [node 3] ICP22 created new session: [4:7571519018645326295:2050]
2025-11-11T17:20:46.363013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519018645326295:2050] [node 3] ICS09 handshake done sender: [4:7571519014350358969:2106] self: [4:7571519018645326287:0] peer: [3:7571519016027322767:0] socket: 513 qp: -1
2025-11-11T17:20:46.363047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519018645326295:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:46.363084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:46.370171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519014350358937:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.370241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:46.370300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519014350358937:2064] [node 6] ICP22 created new session: [4:7571519018645326297:2064]
2025-11-11T17:20:46.370353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519018645326297:2064] [node 6] ICS09 handshake done sender: [4:7571519018645326283:2117] self: [4:7571519018645326278:0] peer: [6:7571519019016322455:0] socket: 557 qp: -1
2025-11-11T17:20:46.370390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519018645326297:2064] [node 6] ICS10 traffic start
2025-11-11T17:20:46.370425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:46.430842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519018645326299:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
2025-11-11T17:20:46.509667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519018645326299:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.521769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519014350358903:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:46.588852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519018645326301:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62835
2025-11-11T17:20:46.331124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519015195474439:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(296160437ns) {Configure:104(296150224ns) {ProcessConfigured:135(12082415ns) {StartInitialHandshake:157(12077758ns) {DropHandshakes:469(11882784ns) {DropIncomingHandshake:441(2236ns) DropOutgoingHandshake:456(395ns)} GenerateSessionVirtualId:988(3046ns) PrepareNewSessionHandshake:476(2994ns) {SwitchToState:178(913ns)}}}}}
2025-11-11T17:20:46.615803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519019016322463:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3451
2025-11-11T17:20:46.341753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441742:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.603677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519015621527958:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:46.638044Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.639171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519014721355066:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519019016322461:2123] held: no
2025-11-11T17:20:46.645359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519019016322461:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:46.581845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519019919722116:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61043
2025-11-11T17:20:46.635650Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:46.637815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519018645326299:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:46.650211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519014350358903:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519018645326299:2121] held: no
2025-11-11T17:20:46.606042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519019919722108:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:46.341802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519019490441742:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.666075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519015621527959:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(166549336ns) {HandleHandshakeStatus:329(166545872ns) {DropIncomingHandshake:441(3169ns) DropOutgoingHandshake:456(19444ns) SwitchToState:178(995ns) ProcessPendingSessionEvents:495(15972229ns) {ForwardSessionEventToSession:612(15909225ns) ForwardSessionEventToSession:612(7361ns) ForwardSessionEventToSession:612(3245ns) ForwardSessionEventToSession:612(8045ns) ForwardSessionEventToSession:612(3273ns) ForwardSessionEventToSession:612(5787ns) ForwardSessionEventToSession:612(6097ns) ForwardSessionEventToSession:612(3465ns) ForwardSessionEventToSession:612(8714ns) ForwardSessionEventToSession:612(3699ns)}}}
2025-11-11T17:20:46.667308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519014721355066:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.354755Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519019490441742:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:46.374676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441755:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.377701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519015195474379:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.389710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441744:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.389754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519019490441744:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.391373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:46.405964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519015195474379:2049] [node 3] ICP22 created new session: [1:7571519019490441764:2049]
2025-11-11T17:20:46.406025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519019490441764:2049] [node 3] ICS09 handshake done sender: [1:7571519019490441742:2111] self: [1:7571519019490441741:0] peer: [3:7571519020322290088:0] socket: 549 qp: -1
2025-11-11T17:20:46.406065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519019490441764:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:46.406112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:46.406616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519015195474379:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(48666628ns) {HandleHandshakeStatus:329(48664716ns) {DropIncomingHandshake:441(907ns) DropOutgoingHandshake:456(642ns) SwitchToState:178(924ns) ProcessPendingSessionEvents:495(109491ns) {ForwardSessionEventToSession:612(56835ns) ForwardSessionEventToSession:612(4456ns) ForwardSessionEventToSession:612(8301ns) ForwardSessionEventToSession:612(3171ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(5671ns) ForwardSessionEventToSession:612(5972ns) ForwardSessionEventToSession:612(3349ns) ForwardSessionEventToSession:612(5572ns) ForwardSessionEventToSession:612(3092ns)}}}
2025-11-11T17:20:46.406780Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519019490441744:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:46.409763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519015195474379:2049] [node 3] ICP18 handshake (actor [1:7571519019490441755:2117]) is too old
2025-11-11T17:20:46.409952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519015195474378:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.410032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:46.410092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519015195474378:2048] [node 2] ICP22 created new session: [1:7571519019490441767:2048]
2025-11-11T17:20:46.410144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519019490441767:2048] [node 2] ICS09 handshake done sender: [1:7571519019490441744:2112] self: [1:7571519019490441743:0] peer: [2:7571519019919722112:0] socket: 550 qp: -1
2025-11-11T17:20:46.413785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519019490441767:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:46.422944Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441749:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.422987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519019490441749:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.423122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:46.478068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519015195474378:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(67888106ns) {HandleHandshakeStatus:329(67885232ns) {DropIncomingHandshake:441(12515ns) DropOutgoingHandshake:456(571ns) SwitchToState:178(693ns) ProcessPendingSessionEvents:495(19972515ns) {ForwardSessionEventToSession:612(19894859ns) ForwardSessionEventToSession:612(11782ns) ForwardSessionEventToSession:612(6487ns) ForwardSessionEventToSession:612(3502ns) ForwardSessionEventToSession:612(6468ns) ForwardSessionEventToSession:612(3418ns) ForwardSessionEventToSession:612(3759ns) ForwardSessionEventToSession:612(6909ns) ForwardSessionEventToSession:612(3816ns) ForwardSessionEventToSession:612(14879ns)}}}
2025-11-11T17:20:46.502559Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519019490441749:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:46.534556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519015195474412:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.534649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:46.534713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519015195474412:2072] [node 5] ICP22 created new session: [1:7571519019490441771:2072]
2025-11-11T17:20:46.534757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519019490441771:2072] [node 5] ICS09 handshake done sender: [1:7571519019490441749:2114] self: [1:7571519019490441748:0] peer: [5:7571519019916495345:0] socket: 551 qp: -1
2025-11-11T17:20:46.534789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519019490441771:2072] [node 5] ICS10 traffic start
2025-11-11T17:20:46.534840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:46.545715Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441756:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.545758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519019490441756:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.546824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519019490441756:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:46.569968Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519019490441759:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:46.569995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519019490441759:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:46.577117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519015195474443:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.583445Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519019490441759:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:46.589245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519015195474439:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:46.589315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:46.589362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519015195474439:2068] [node 6] ICP22 created new session: [1:7571519019490441775:2068]
2025-11-11T17:20:46.589409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519019490441775:2068] [node 6] ICS09 handshake done sender: [1:7571519019490441759:2119] self: [1:7571519019490441758:0] peer: [6:7571519019016322462:0] socket: 544 qp: -1
2025-11-11T17:20:46.589445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519019490441775:2068] [node 6] ICS10 traffic start
2025-11-11T17:20:46.589484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:46.619408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:46.633734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519015195474443:2071] [node 4] ICP22 created new session: [1:7571519019490441777:2071]
2025-11-11T17:20:46.633799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519019490441777:2071] [node 4] ICS09 handshake done sender: [1:7571519019490441756:2118] self: [1:7571519019490441753:0] peer: [4:7571519018645326300:0] socket: 530 qp: -1
2025-11-11T17:20:46.633846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519019490441777:2071] [node 4] ICS10 traffic start
2025-11-11T17:20:46.633920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:46.634386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519015195474443:2071] [node 4] ICP35 event processing took too much time PendingConnection:278(57068250ns) {HandleHandshakeStatus:329(57064532ns) {DropIncomingHandshake:441(15871ns) DropOutgoingHandshake:456(452ns) SwitchToState:178(810ns) ProcessPendingSessionEvents:495(110448ns) {ForwardSessionEventToSession:612(55912ns) ForwardSessionEventToSession:612(6641ns) ForwardSessionEventToSession:612(3179ns) ForwardSessionEventToSession:612(3652ns) ForwardSessionEventToSession:612(5290ns) ForwardSessionEventToSession:612(2441ns) ForwardSessionEventToSession:612(13329ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(2363ns) ForwardSessionEventToSession:612(4451ns)}}}
2025-11-11T17:20:46.662379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519015195474439:2068] [node 6] ICP35 event processing took too much time PendingConnection:278(49845082ns) {HandleHandshakeStatus:329(49842113ns) {DropIncomingHandshake:441(14822ns) DropOutgoingHandshake:456(402ns) SwitchToState:178(1357ns) ProcessPendingSessionEvents:495(49445735ns) {ForwardSessionEventToSession:612(20053389ns) ForwardSessionEventToSession:612(6476991ns) ForwardSessionEventToSession:612(5654860ns) ForwardSessionEventToSession:612(10187ns) ForwardSessionEventToSession:612(7742447ns) ForwardSessionEventToSession:612(4285ns) ForwardSessionEventToSession:612(3748ns) ForwardSessionEventToSession:612(5284709ns) ForwardSessionEventToSession:612(3911ns) ForwardSessionEventToSession:612(4178700ns)}}}
2025-11-11T17:20:46.606308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519015624754722:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.676880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:46.682440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519014721355066:2048] [node 1] ICP22 created new session: [6:7571519019016322464:2048]
2025-11-11T17:20:46.686409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519014350358903:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19761304ns) {HandleHandshakeStatus:395(19757965ns) {LogHandshakeFail:472(7511380ns) {UpdateErrorStateLog:544(7354ns)} DropOutgoingHandshake:456(937ns) ProcessPendingSessionEvents:495(11699685ns) {DropSessionEvent:512(20163ns) DropSessionEvent:512(6124ns) DropSessionEvent:512(4475ns) DropSessionEvent:512(5358ns) DropSessionEvent:512(5537ns) DropSessionEvent:512(33530ns) DropSessionEvent:512(9464ns) DropSessionEvent:512(15012ns) DropSessionEvent:512(1063002ns) DropSessionEvent:512(9932ns)}}}
2025-11-11T17:20:46.686519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519019016322464:2048] [node 1] ICS09 handshake done sender: [6:7571519019016322461:2123] self: [6:7571519019016322462:0] peer: [1:7571519019490441758:0] socket: 572 qp: -1
2025-11-11T17:20:46.686589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519019016322464:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:46.686649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:46.686909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519014721355066:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24837677ns) {HandleHandshakeStatus:329(19522025ns) {DropIncomingHandshake:441(605ns) DropOutgoingHandshake:456(2320ns) SwitchToState:178(3396ns) ProcessPendingSessionEvents:495(523ns)}}
2025-11-11T17:20:46.606350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:46.692927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519014350358903:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:46.693052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:46.693110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519014350358903:2048] [node 1] ICP22 created new session: [4:7571519018645326302:2048]
2025-11-11T17:20:46.693159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519018645326302:2048] [node 1] ICS09 handshake done sender: [4:7571519018645326299:2121] self: [4:7571519018645326300:0] peer: [1:7571519019490441753:0] socket: 531 qp: -1
2025-11-11T17:20:46.693202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519018645326302:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:46.693249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:46.606402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519015624754722:2048] [node 1] ICP22 created new session: [2:7571519019919722121:2048]
2025-11-11T17:20:46.606455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519019919722121:2048] [node 1] ICS09 handshake done sender: [2:7571519019919722108:2118] self: [2:7571519019919722112:0] peer: [1:7571519019490441743:0] socket: 552 qp: -1
2025-11-11T17:20:46.606487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519019919722121:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:46.606550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:46.798275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:46.798496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519020322290098:2048] [node 1] ICS01 socket: 506 reason# EndOfStream
2025-11-11T17:20:46.798540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519016027322699:2048] [node 1] ICP30 unregister session Session# [3:7571519020322290098:2048] VirtualId# [3:7571519020322290088:0]
2025-11-11T17:20:46.798577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519020322290098:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.798649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:46.798688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:46.799941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:46.800167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519019919722121:2048] [node 1] ICS01 socket: 552 reason# EndOfStream
2025-11-11T17:20:46.800208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519015624754722:2048] [node 1] ICP30 unregister session Session# [2:7571519019919722121:2048] VirtualId# [2:7571519019919722112:0]
2025-11-11T17:20:46.800239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519019919722121:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.800329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:46.800389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:46.805734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:46.805967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519019016322464:2048] [node 1] ICS01 socket: 572 reason# EndOfStream
2025-11-11T17:20:46.806011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519014721355066:2048] [node 1] ICP30 unregister session Session# [6:7571519019016322464:2048] VirtualId# [6:7571519019016322462:0]
2025-11-11T17:20:46.806042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519019016322464:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.806123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:46.806161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:46.806473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:46.806678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519019916495357:2048] [node 1] ICS01 socket: 554 reason# EndOfStream
2025-11-11T17:20:46.806708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519015621527958:2048] [node 1] ICP30 unregister session Session# [5:7571519019916495357:2048] VirtualId# [5:7571519019916495345:0]
2025-11-11T17:20:46.806733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519019916495357:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.806785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:46.806821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:46.809760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:46.809934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519018645326302:2048] [node 1] ICS01 socket: 531 reason# EndOfStream
2025-11-11T17:20:46.809969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519014350358903:2048] [node 1] ICP30 unregister session Session# [4:7571519018645326302:2048] VirtualId# [4:7571519018645326300:0]
2025-11-11T17:20:46.810004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519018645326302:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.810083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:46.810118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:46.928681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:46.928897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519020322290096:2049] [node 2] ICS01 socket: 542 reason# EndOfStream
2025-11-11T17:20:46.928937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519016027322700:2049] [node 2] ICP30 unregister session Session# [3:7571519020322290096:2049] VirtualId# [3:7571519020322290090:0]
2025-11-11T17:20:46.928971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519020322290096:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.929047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:46.929088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:46.957773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:46.957780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:46.958034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519019916495355:2049] [node 2] ICS07 socket disconnect 562 reason# EndOfStream
2025-11-11T17:20:46.958066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519019916495355:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.958163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:46.958206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519019916495355:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:46.958108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519019016322448:2049] [node 2] ICS07 socket disconnect 526 reason# EndOfStream
2025-11-11T17:20:46.958142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519019016322448:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.958220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:46.958266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519019016322448:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:46.959173Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address
2025-11-11T17:20:46.964812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519014721355067:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:46.965709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:46.969767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519018645326291:2049] [node 2] ICS01 socket: 523 reason# EndOfStream
2025-11-11T17:20:46.979155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519014350358904:2049] [node 2] ICP30 unregister session Session# [4:7571519018645326291:2049] VirtualId# [4:7571519018645326282:0]
2025-11-11T17:20:46.979217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519018645326291:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:46.979324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:46.979375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:46.997854Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address
2025-11-11T17:20:46.998203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519015621527959:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:46.998262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519019916495355:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:46.998296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519015621527959:2049] [node 2] ICP30 unregister session Session# [5:7571519019916495355:2049] VirtualId# [5:7571519019916495346:0]
2025-11-11T17:20:46.998350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:46.998383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519015621527959:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:46.998397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:46.981744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519019016322448:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:47.009778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519015621527959:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:46.981834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519014721355067:2049] [node 2] ICP30 unregister session Session# [6:7571519019016322448:2049] VirtualId# [6:7571519019016322445:0]
2025-11-11T17:20:46.981923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:46.981990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519014721355067:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:46.982013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:46.990148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [6:7571519014721355067:2049] [node 2] ICP35 event processing took too much time StateWork:293(30482648ns) {HandleHandshakeStatus:395(25283207ns) {LogHandshakeFail:472(17673ns) {UpdateErrorStateLog:544(4111ns)} DropOutgoingHandshake:456(921ns) ProcessPendingSessionEvents:495(286ns) UnregisterSession:538(80486ns) {ProcessPendingSessionEvents:495(413ns) SwitchToInitialState:201(5224ns) {SwitchToState:178(3805ns)}} TransitToErrorState:752(7904988ns) {SwitchToState:178(7047028ns) ProcessPendingSessionEvents:495(912ns) DropHandshakes:469(2891ns) {DropIncomingHandshake:441(850ns) DropOutgoingHandshake:456(452ns)}}}}
2025-11-11T17:20:47.005116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519014721355067:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:47.105269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:47.105484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519019016322456:2050] [node 3] ICS01 socket: 527 reason# EndOfStream
2025-11-11T17:20:47.105526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519014721355068:2050] [node 3] ICP30 unregister session Session# [6:7571519019016322456:2050] VirtualId# [6:7571519019016322441:0]
2025-11-11T17:20:47.105559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519019016322456:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:47.105662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:47.105703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:47.105755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:47.105930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519019916495353:2050] [node 3] ICS01 socket: 561 reason# EndOfStream
2025-11-11T17:20:47.105970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519015621527960:2050] [node 3] ICP30 unregister session Session# [5:7571519019916495353:2050] VirtualId# [5:7571519019916495347:0]
2025-11-11T17:20:47.106011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519019916495353:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:47.106105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:47.106147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:47.110230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:47.110475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519018645326295:2050] [node 3] ICS01 socket: 513 reason# EndOfStream
2025-11-11T17:20:47.110517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519014350358905:2050] [node 3] ICP30 unregister session Session# [4:7571519018645326295:2050] VirtualId# [4:7571519018645326287:0]
2025-11-11T17:20:47.110553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519018645326295:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:47.110668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:47.110719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:47.269791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:47.270009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519019016322459:2061] [node 4] ICS01 socket: 558 reason# EndOfStream
2025-11-11T17:20:47.270051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519014721355080:2061] [node 4] ICP30 unregister session Session# [6:7571519019016322459:2061] VirtualId# [6:7571519019016322455:0]
2025-11-11T17:20:47.270083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519019016322459:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:47.270170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:47.270214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:47.270312Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:47.270471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519019916495348:2059] [node 4] ICS01 socket: 518 reason# EndOfStream
2025-11-11T17:20:47.270500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519015621527971:2059] [node 4] ICP30 unregister session Session# [5:7571519019916495348:2059] VirtualId# [5:7571519019916495334:0]
2025-11-11T17:20:47.270524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519019916495348:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:47.270577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:47.270608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:47.353765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:47.353993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519019016322453:2065] [node 5] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-11T17:20:47.354022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519019016322453:2065] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:47.354133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:47.354179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519019016322453:2065] [node 5] ICS15 start handshake
2025-11-11T17:20:47.355052Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8774) Couldn't connect to any resolved address
2025-11-11T17:20:47.355279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519014721355089:2065] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8774) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:47.355327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519019016322453:2065] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:47.355365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519014721355089:2065] [node 5] ICP30 unregister session Session# [6:7571519019016322453:2065] VirtualId# [6:7571519019016322442:0]
2025-11-11T17:20:47.355418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:20:47.355447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519014721355089:2065] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8774) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:47.355461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8774) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:47.365827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519014721355089:2065] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:47.584007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519024133011533:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:47.584061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519024133011534:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:47.678756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519022960985139:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:47.678805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519022960985140:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:47.641843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519022574229727:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:47.641895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519022574229728:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:47.755998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519022852402491:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:47.756047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519022852402492:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:47.756079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519022852402493:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:47.837907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519021015036715:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:47.837951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519021015036716:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:47.837979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519021015036717:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:47.889956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519020828326639:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:47.890000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519020828326640:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:47.890029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519020828326641:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:48.190515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519024133011534:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(95500512ns) {Configure:104(95494693ns) {ProcessConfigured:135(20403734ns) {StartInitialHandshake:157(20399774ns) {DropHandshakes:469(3356ns) {DropIncomingHandshake:441(537ns) DropOutgoingHandshake:456(1065ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:476(5679ns) {SwitchToState:178(3078ns)}}}}}
2025-11-11T17:20:48.218730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519026869197046:2062] [node 6] ICP01 ready to work
2025-11-11T17:20:48.242754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519026869197046:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23698367ns) {Configure:104(23693919ns) {ProcessConfigured:135(82155ns) {StartInitialHandshake:157(79676ns) {DropHandshakes:469(2240ns) {DropIncomingHandshake:441(613ns) DropOutgoingHandshake:456(477ns)} GenerateSessionVirtualId:988(660ns) PrepareNewSessionHandshake:476(1879ns) {SwitchToState:178(666ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.210641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519028427978861:2071] [node 4] ICP01 ready to work
2025-11-11T17:20:48.218124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519027255952459:2063] [node 6] ICP01 ready to work
2025-11-11T17:20:48.259074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519022574229728:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(97004584ns) {Configure:104(96999193ns) {ProcessConfigured:135(67725ns) {StartInitialHandshake:157(65700ns) {DropHandshakes:469(2466ns) {DropIncomingHandshake:441(540ns) DropOutgoingHandshake:456(389ns)} GenerateSessionVirtualId:988(884ns) PrepareNewSessionHandshake:476(1341ns) {SwitchToState:178(546ns)}}}}}
2025-11-11T17:20:48.220094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519027255952458:2066] [node 4] ICP01 ready to work
2025-11-11T17:20:48.257762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519026869197047:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.211494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519024133011533:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(116722134ns) {Configure:104(116716497ns) {ProcessConfigured:135(2813002ns) {StartInitialHandshake:157(2810086ns) {DropHandshakes:469(2741ns) {DropIncomingHandshake:441(670ns) DropOutgoingHandshake:456(511ns)} GenerateSessionVirtualId:988(701ns) PrepareNewSessionHandshake:476(3844ns) {SwitchToState:178(1216ns)}}}}}
2025-11-11T17:20:48.251160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519027255952461:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11865
2025-11-11T17:20:48.254157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519027255952462:2061] [node 5] ICP01 ready to work
2025-11-11T17:20:48.295991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123293947:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.295530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519026869197054:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20503
2025-11-11T17:20:48.217626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519028427978863:2072] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.320289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369799:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.275538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519028427978861:2071] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53913877ns) {Configure:104(53907853ns) {ProcessConfigured:135(88426ns) {StartInitialHandshake:157(86247ns) {DropHandshakes:469(3228ns) {DropIncomingHandshake:441(494ns) DropOutgoingHandshake:456(714ns)} GenerateSessionVirtualId:988(816ns) PrepareNewSessionHandshake:476(2526ns) {SwitchToState:178(1274ns)}}}}}
2025-11-11T17:20:48.293955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519028427978871:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65064
2025-11-11T17:20:48.320512Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978871:2081] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.321727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978862:2078] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.321774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519028427978862:2078] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.330721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519026869197054:2072] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.282645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519022960985139:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(183717893ns) {Configure:104(168765105ns) {ProcessConfigured:135(74642ns) {StartInitialHandshake:157(69228ns) {DropHandshakes:469(2567ns) {DropIncomingHandshake:441(612ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(1287ns) PrepareNewSessionHandshake:476(2319ns) {SwitchToState:178(1097ns)}}}}}
2025-11-11T17:20:48.337514Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519028427978862:2078] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:48.301651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123293946:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.286020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519027255952459:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(43605105ns) {Configure:104(43600192ns) {ProcessConfigured:135(41901413ns) {StartInitialHandshake:157(41898516ns) {DropHandshakes:469(2768ns) {DropIncomingHandshake:441(514ns) DropOutgoingHandshake:456(435ns)} GenerateSessionVirtualId:988(613ns) PrepareNewSessionHandshake:476(2705ns) {SwitchToState:178(880ns)}}}}}
2025-11-11T17:20:48.306325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519027255952461:2070] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.306474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519022960985139:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.352891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519027255952473:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11865
2025-11-11T17:20:48.346323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519028427978863:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(98659553ns) {Configure:104(98655052ns) {ProcessConfigured:135(65559ns) {StartInitialHandshake:157(63442ns) {DropHandshakes:469(1411ns) {DropIncomingHandshake:441(341ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:476(1183ns) {SwitchToState:178(615ns)}}}}}
2025-11-11T17:20:48.362621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519024133011534:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.362708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:48.362761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519024133011534:2049] [node 3] ICP22 created new session: [1:7571519028427978876:2049]
2025-11-11T17:20:48.362807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519028427978876:2049] [node 3] ICS09 handshake done sender: [1:7571519028427978862:2078] self: [1:7571519028427978860:0] peer: [3:7571519027255952469:0] socket: 510 qp: -1
2025-11-11T17:20:48.362847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519028427978876:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:48.362897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:48.337124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519025123293947:2059] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.365980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519027255952458:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(64360493ns) {Configure:104(64353037ns) {ProcessConfigured:135(94641ns) {StartInitialHandshake:157(91385ns) {DropHandshakes:469(2708ns) {DropIncomingHandshake:441(513ns) DropOutgoingHandshake:456(717ns)} GenerateSessionVirtualId:988(611ns) PrepareNewSessionHandshake:476(2166ns) {SwitchToState:178(644ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.359046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519020828326641:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.359270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519020828326641:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21744698ns) {Configure:104(21742796ns) {ProcessConfigured:135(150808ns) {IncomingHandshake:261(120051ns) {DropIncomingHandshake:441(450ns) PrepareNewSessionHandshake:476(1967ns) {SwitchToState:178(684ns)} GenerateSessionVirtualId:988(5665ns) IssueIncomingHandshakeReply:193(86635ns) {SwitchToState:178(189ns)}}}}}
2025-11-11T17:20:48.364064Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519025123293946:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.386567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519022574229727:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(232490197ns) {Configure:104(232485322ns) {ProcessConfigured:135(5722380ns) {StartInitialHandshake:157(5718571ns) {DropHandshakes:469(3442ns) {DropIncomingHandshake:441(600ns) DropOutgoingHandshake:456(593ns)} GenerateSessionVirtualId:988(866ns) PrepareNewSessionHandshake:476(5373ns) {SwitchToState:178(2683ns)}}}}}
2025-11-11T17:20:48.390266Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519026869197049:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.390297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519026869197049:2070] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.391637Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519026869197049:2070] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:48.392087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519026869197046:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.386605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519020828326640:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.377845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519027255952464:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11865
2025-11-11T17:20:48.392147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.405908Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519027255952464:2072] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.413628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519021015036717:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(122481854ns) {Configure:104(122479187ns) {ProcessConfigured:135(43962202ns) {StartInitialHandshake:157(43958778ns) {DropHandshakes:469(3138ns) {DropIncomingHandshake:441(634ns) DropOutgoingHandshake:456(503ns)} GenerateSessionVirtualId:988(537ns) PrepareNewSessionHandshake:476(4873ns) {SwitchToState:178(2255ns)}}}}}
2025-11-11T17:20:48.413017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519020828326640:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22446497ns) {Configure:104(22445029ns) {ProcessConfigured:135(127310ns) {IncomingHandshake:261(99135ns) {DropIncomingHandshake:441(377ns) PrepareNewSessionHandshake:476(1132ns) {SwitchToState:178(311ns)} GenerateSessionVirtualId:988(3975ns) IssueIncomingHandshakeReply:193(69721ns) {SwitchToState:178(148ns)}}}}}
2025-11-11T17:20:48.425651Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978866:2079] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.423655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369798:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.509647Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519027147369799:2059] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.392195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519026869197046:2062] [node 6] ICP22 created new session: [2:7571519026869197060:2062]
2025-11-11T17:20:48.442175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123293952:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.452550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519025310004033:2064] [node 4] ICP01 ready to work
2025-11-11T17:20:48.406362Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519027255952461:2070] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:48.392259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519026869197060:2062] [node 6] ICS09 handshake done sender: [2:7571519026869197049:2070] self: [2:7571519026869197048:0] peer: [6:7571519025123293949:0] socket: 513 qp: -1
2025-11-11T17:20:48.425724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519028427978866:2079] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.521563Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519027147369798:2058] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.483580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519028427978866:2079] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.460067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519025123293958:2066] [node 5] ICP01 ready to work
2025-11-11T17:20:48.425840Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:48.433959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519027255952463:2071] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.434004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519027255952463:2071] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.437676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519022960985139:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519027255952461:2070] held: no
2025-11-11T17:20:48.462276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519022960985139:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.462336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:48.462379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519022960985139:2048] [node 1] ICP22 created new session: [3:7571519027255952485:2048]
2025-11-11T17:20:48.462429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519027255952485:2048] [node 1] ICS09 handshake done sender: [3:7571519027255952461:2070] self: [3:7571519027255952469:0] peer: [1:7571519028427978860:0] socket: 512 qp: -1
2025-11-11T17:20:48.462466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519027255952485:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:48.462503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:48.474945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519027255952462:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(184627030ns) {Configure:104(184618578ns) {ProcessConfigured:135(21380891ns) {StartInitialHandshake:157(21378385ns) {DropHandshakes:469(3496ns) {DropIncomingHandshake:441(1138ns) DropOutgoingHandshake:456(341ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:476(4609ns) {SwitchToState:178(2355ns)}}}}}
2025-11-11T17:20:48.476775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519022960985140:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.477024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519027255952476:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11865
2025-11-11T17:20:48.518522Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519027255952463:2071] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:48.522225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519022960985139:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(59843799ns) {HandleHandshakeStatus:329(59841407ns) {DropIncomingHandshake:441(908ns) DropOutgoingHandshake:456(388ns) SwitchToState:178(394ns) ProcessPendingSessionEvents:495(80050ns) {ForwardSessionEventToSession:612(77409ns)}}}
2025-11-11T17:20:48.634398Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519027255952476:2078] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.635684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519027255952459:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.635726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:48.636158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519027255952525:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11865
2025-11-11T17:20:48.655455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519027255952459:2063] [node 6] ICP22 created new session: [3:7571519027255952527:2063]
2025-11-11T17:20:48.518843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004032:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.546189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519027147369825:2081] [node 6] ICP01 ready to work
2025-11-11T17:20:48.392289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519026869197060:2062] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.619289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519020828326639:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(209306346ns) {Configure:104(209302628ns) {ProcessConfigured:135(79252ns) {StartInitialHandshake:157(76978ns) {DropHandshakes:469(3628ns) {DropIncomingHandshake:441(579ns) DropOutgoingHandshake:456(479ns)} GenerateSessionVirtualId:988(707ns) PrepareNewSessionHandshake:476(1837ns) {SwitchToState:178(521ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.484180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519024133011533:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.392325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:48.570156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519025310004038:2062] [node 6] ICP01 ready to work
2025-11-11T17:20:48.679780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519027255952527:2063] [node 6] ICS09 handshake done sender: [3:7571519027255952463:2071] self: [3:7571519027255952460:0] peer: [6:7571519025123293948:0] socket: 518 qp: -1
2025-11-11T17:20:48.661970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519027147369853:2070] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.484237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:48.637187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519025123293946:2058] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:48.659291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519025123294011:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.631608Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519025310004032:2066] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.392782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519022574229727:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.670370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519022852402492:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(263991336ns) {Configure:104(263986543ns) {ProcessConfigured:135(12430040ns) {StartInitialHandshake:157(12425449ns) {DropHandshakes:469(2631ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(1041ns) PrepareNewSessionHandshake:476(5597ns) {SwitchToState:178(2237ns)}}}}}
2025-11-11T17:20:48.693708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519027255952527:2063] [node 6] ICS10 traffic start
2025-11-11T17:20:48.670870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123294008:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.432100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519026869197070:2081] [node 5] ICP01 ready to work
2025-11-11T17:20:48.722036Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519027255952464:2072] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.484295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519024133011533:2048] [node 2] ICP22 created new session: [1:7571519028427978892:2048]
2025-11-11T17:20:48.631975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004079:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.719286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519022852402493:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.676479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123294009:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.484342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519028427978892:2048] [node 2] ICS09 handshake done sender: [1:7571519028427978866:2079] self: [1:7571519028427978865:0] peer: [2:7571519026869197057:0] socket: 511 qp: -1
2025-11-11T17:20:48.667570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519025310004033:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(157713449ns) {Configure:104(145692657ns) {ProcessConfigured:135(80803ns) {StartInitialHandshake:157(78609ns) {DropHandshakes:469(2729ns) {DropIncomingHandshake:441(706ns) DropOutgoingHandshake:456(665ns)} GenerateSessionVirtualId:988(1114ns) PrepareNewSessionHandshake:476(1519ns) {SwitchToState:178(554ns)}}}}}
2025-11-11T17:20:48.719526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519022852402493:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(205548521ns) {Configure:104(205546572ns) {ProcessConfigured:135(121561ns) {IncomingHandshake:261(88797ns) {DropIncomingHandshake:441(246ns) PrepareNewSessionHandshake:476(1946ns) {SwitchToState:178(691ns)} GenerateSessionVirtualId:988(6050ns) IssueIncomingHandshakeReply:193(52849ns) {SwitchToState:178(192ns)}}}}}
2025-11-11T17:20:48.566379Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519026869197052:2071] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.732172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:48.676704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123294010:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.566432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519026869197052:2071] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.677808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519025310004079:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.757827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519027255952471:2074] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.676802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519020828326640:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.484379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519028427978892:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:48.681779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004031:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.769664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519027147369827:2085] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.769725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519027147369827:2085] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.802497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369862:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.802637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369863:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.804839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519027147369862:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.804937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519027147369799:2059] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:48.805205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519022852402493:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.805278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:48.805323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519022852402493:2050] [node 3] ICP22 created new session: [4:7571519027147369864:2050]
2025-11-11T17:20:48.805369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519027147369864:2050] [node 3] ICS09 handshake done sender: [4:7571519027147369799:2059] self: [4:7571519027147369860:0] peer: [3:7571519027255952470:0] socket: 525 qp: -1
2025-11-11T17:20:48.805409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519027147369864:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:48.805444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:48.833768Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519027147369827:2085] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:48.855581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519027147369825:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.855679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:48.855727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519027147369825:2081] [node 6] ICP22 created new session: [4:7571519027147369870:2081]
2025-11-11T17:20:48.855772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519027147369870:2081] [node 6] ICS09 handshake done sender: [4:7571519027147369827:2085] self: [4:7571519027147369826:0] peer: [6:7571519025123294017:0] socket: 539 qp: -1
2025-11-11T17:20:48.855808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519027147369870:2081] [node 6] ICS10 traffic start
2025-11-11T17:20:48.855844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:48.860682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.874284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519027147369853:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(180217146ns) {Configure:104(180205090ns) {ProcessConfigured:135(64174ns) {StartInitialHandshake:157(61790ns) {DropHandshakes:469(761ns) {DropIncomingHandshake:441(266ns) DropOutgoingHandshake:456(125ns)} GenerateSessionVirtualId:988(690ns) PrepareNewSessionHandshake:476(2165ns) {SwitchToState:178(813ns)}}}}}
2025-11-11T17:20:48.873846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519022852402491:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(442214516ns) {Configure:104(442204275ns) {ProcessConfigured:135(20811019ns) {StartInitialHandshake:157(20807763ns) {DropHandshakes:469(4319ns) {DropIncomingHandshake:441(1403ns) DropOutgoingHandshake:456(753ns)} GenerateSessionVirtualId:988(1425ns) PrepareNewSessionHandshake:476(5152ns) {SwitchToState:178(1977ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.757869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519027255952471:2074] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.632519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519026869197047:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(355234923ns) {Configure:104(355228485ns) {ProcessConfigured:135(16226726ns) {StartInitialHandshake:157(16224255ns) {DropHandshakes:469(2389ns) {DropIncomingHandshake:441(471ns) DropOutgoingHandshake:456(494ns)} GenerateSessionVirtualId:988(758ns) PrepareNewSessionHandshake:476(3497ns) {SwitchToState:178(1899ns)}}}}}
2025-11-11T17:20:48.890909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519022852402491:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.484431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:48.641924Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519026869197052:2071] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:48.698303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519026869197077:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20503
2025-11-11T17:20:48.699389Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519026869197054:2072] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:48.699674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519022574229727:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.699775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:48.699826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519022574229727:2048] [node 1] ICP22 created new session: [2:7571519026869197111:2048]
2025-11-11T17:20:48.699871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519026869197111:2048] [node 1] ICS09 handshake done sender: [2:7571519026869197054:2072] self: [2:7571519026869197057:0] peer: [1:7571519028427978865:0] socket: 519 qp: -1
2025-11-11T17:20:48.699910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519026869197111:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:48.699948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:48.717847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519022574229728:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.722057Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:48.737717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519022574229727:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:48.742179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519026869197114:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20503
2025-11-11T17:20:48.750109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519026869197113:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20503
2025-11-11T17:20:48.757745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:48.769868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519022574229728:2049] [node 3] ICP22 created new session: [2:7571519026869197115:2049]
2025-11-11T17:20:48.789821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519026869197115:2049] [node 3] ICS09 handshake done sender: [2:7571519026869197052:2071] self: [2:7571519026869197051:0] peer: [3:7571519027255952492:0] socket: 515 qp: -1
2025-11-11T17:20:48.797824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519026869197115:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:48.802845Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519026869197072:2082] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.802873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519026869197072:2082] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.817782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519026869197114:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.819930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:48.825781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519026869197113:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.825893Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519026869197072:2082] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:48.854573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519022574229728:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(136448211ns) {HandleHandshakeStatus:329(136443818ns) {DropIncomingHandshake:441(391ns) DropOutgoingHandshake:456(907ns) SwitchToState:178(912ns) ProcessPendingSessionEvents:495(8502138ns) {ForwardSessionEventToSession:612(8439423ns) ForwardSessionEventToSession:612(12244ns) ForwardSessionEventToSession:612(5748ns) ForwardSessionEventToSession:612(3238ns) ForwardSessionEventToSession:612(5723ns) ForwardSessionEventToSession:612(3543ns) ForwardSessionEventToSession:612(2505ns) ForwardSessionEventToSession:612(5615ns) ForwardSessionEventToSession:612(3555ns) ForwardSessionEventToSession:612(5300ns)}}}
2025-11-11T17:20:48.854797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519022574229728:2049] [node 3] ICP18 handshake (actor [2:7571519026869197113:2117]) is too old
2025-11-11T17:20:48.860516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519026869197070:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.877718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:48.885744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519026869197070:2081] [node 5] ICP22 created new session: [2:7571519026869197119:2081]
2025-11-11T17:20:48.893844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519026869197119:2081] [node 5] ICS09 handshake done sender: [2:7571519026869197072:2082] self: [2:7571519026869197071:0] peer: [5:7571519025310004093:0] socket: 535 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.676854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.705985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519021015036716:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(274220140ns) {Configure:104(274213283ns) {ProcessConfigured:135(24014592ns) {StartInitialHandshake:157(24009771ns) {DropHandshakes:469(3714ns) {DropIncomingHandshake:441(716ns) DropOutgoingHandshake:456(683ns)} GenerateSessionVirtualId:988(1128ns) PrepareNewSessionHandshake:476(5129ns) {SwitchToState:178(2429ns)}}}}}
2025-11-11T17:20:48.766770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519022960985140:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.905991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519026869197119:2081] [node 5] ICS10 traffic start
2025-11-11T17:20:48.676912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519020828326640:2049] [node 2] ICP22 created new session: [6:7571519025123294013:2049]
2025-11-11T17:20:48.916799Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519027147369861:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.722334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519021015036717:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.508311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519028427978886:2084] [node 6] ICP01 ready to work
2025-11-11T17:20:48.787003Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519027255952471:2074] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:48.787078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:48.790151Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519027255952489:2087] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.790194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519027255952489:2087] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.791708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519022960985140:2049] [node 2] ICP22 created new session: [3:7571519027255952531:2049]
2025-11-11T17:20:48.805886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519027255952531:2049] [node 2] ICS09 handshake done sender: [3:7571519027255952464:2072] self: [3:7571519027255952492:0] peer: [2:7571519026869197051:0] socket: 516 qp: -1
2025-11-11T17:20:48.810315Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519027255952489:2087] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:48.810565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519027255952458:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.810614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:48.810671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519027255952458:2066] [node 4] ICP22 created new session: [3:7571519027255952533:2066]
2025-11-11T17:20:48.810720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519027255952533:2066] [node 4] ICS09 handshake done sender: [3:7571519027255952471:2074] self: [3:7571519027255952470:0] peer: [4:7571519027147369860:0] socket: 524 qp: -1
2025-11-11T17:20:48.810755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519027255952533:2066] [node 4] ICS10 traffic start
2025-11-11T17:20:48.810790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:48.811259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519027255952462:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.811306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:48.811344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519027255952462:2061] [node 5] ICP22 created new session: [3:7571519027255952535:2061]
2025-11-11T17:20:48.514814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519028427978888:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65064
2025-11-11T17:20:48.613990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978888:2090] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.620153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519028427978916:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65064
2025-11-11T17:20:48.658848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519024133011533:2048] [node 2] ICP18 handshake (actor [1:7571519028427978888:2090]) is too old
2025-11-11T17:20:48.675832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [1:7571519024133011533:2048] [node 2] ICP35 event processing took too much time StateWork:293(25535470ns) {IncomingHandshake:261(25530218ns)}
2025-11-11T17:20:48.702197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519028427978917:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65064
2025-11-11T17:20:48.722566Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978916:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.829671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978873:2082] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.829734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519028427978873:2082] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.839945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519028427978873:2082] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:48.876046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519028427978920:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65064
2025-11-11T17:20:48.880391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519028427978863:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.882702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519028427978886:2084] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(267217957ns) {Configure:104(267208714ns) {ProcessConfigured:135(8701032ns) {StartInitialHandshake:157(8698204ns) {DropHandshakes:469(3399ns) {DropIncomingHandshake:441(747ns) DropOutgoingHandshake:456(689ns)} GenerateSessionVirtualId:988(1337ns) PrepareNewSessionHandshake:476(2659ns) {SwitchToState:178(1150ns)}}}}}
2025-11-11T17:20:48.883346Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978917:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.886492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:48.891027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519028427978863:2072] [node 5] ICP22 created new session: [1:7571519028427978921:2072]
2025-11-11T17:20:48.921835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519028427978921:2072] [node 5] ICS09 handshake done sender: [1:7571519028427978873:2082] self: [1:7571519028427978872:0] peer: [5:7571519025310004094:0] socket: 528 qp: -1
2025-11-11T17:20:48.921910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978869:2080] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.921943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519028427978869:2080] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.923129Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519028427978869:2080] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:48.923364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519028427978861:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.923457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:48.923554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519028427978861:2071] [node 4] ICP22 created new session: [1:7571519028427978926:2071]
2025-11-11T17:20:48.923610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519028427978926:2071] [node 4] ICS09 handshake done sender: [1:7571519028427978869:2080] self: [1:7571519028427978868:0] peer: [4:7571519027147369877:0] socket: 517 qp: -1
2025-11-11T17:20:48.923660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519028427978926:2071] [node 4] ICS10 traffic start
2025-11-11T17:20:48.923716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:48.933968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519028427978921:2072] [node 5] ICS10 traffic start
2025-11-11T17:20:48.934033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:48.934519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519028427978863:2072] [node 5] ICP35 event processing took too much time PendingConnection:278(53933403ns) {HandleHandshakeStatus:329(53928926ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(955ns) SwitchToState:178(686ns) ProcessPendingSessionEvents:495(110650ns) {ForwardSessionEventToSession:612(56310ns) ForwardSessionEventToSession:612(5974ns) ForwardSessionEventToSession:612(3268ns) ForwardSessionEventToSession:612(5369ns) ForwardSessionEventToSession:612(3107ns) ForwardSessionEventToSession:612(5643ns) ForwardSessionEventToSession:612(8109ns) ForwardSessionEventToSession:612(1770ns) ForwardSessionEventToSession:612(2493ns) ForwardSessionEventToSession:612(5926ns)}}}
2025-11-11T17:20:48.934639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519028427978863:2072] [node 5] ICP18 handshake (actor [1:7571519028427978917:2117]) is too old
2025-11-11T17:20:48.967729Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978920:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.676969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519025123294013:2049] [node 2] ICS09 handshake done sender: [6:7571519025123293946:2058] self: [6:7571519025123293949:0] peer: [2:7571519026869197048:0] socket: 514 qp: -1
2025-11-11T17:20:48.769674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519025310004031:2065] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.906076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:48.811383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519027255952535:2061] [node 5] ICS09 handshake done sender: [3:7571519027255952489:2087] self: [3:7571519027255952483:0] peer: [5:7571519025310004092:0] socket: 540 qp: -1
2025-11-11T17:20:48.929843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519022852402493:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(52632390ns) {HandleHandshakeStatus:329(52629920ns) {DropIncomingHandshake:441(1612ns) DropOutgoingHandshake:456(816ns) SwitchToState:178(732ns) ProcessPendingSessionEvents:495(52254895ns) {ForwardSessionEventToSession:612(72538ns) ForwardSessionEventToSession:612(20000381ns) ForwardSessionEventToSession:612(12755ns) ForwardSessionEventToSession:612(4249ns) ForwardSessionEventToSession:612(8333869ns) ForwardSessionEventToSession:612(5989ns) ForwardSessionEventToSession:612(4043ns) ForwardSessionEventToSession:612(388805ns) ForwardSessionEventToSession:612(3628ns) ForwardSessionEventToSession:612(15705143ns)}}}
2025-11-11T17:20:48.935265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519022852402492:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.935640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369878:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.937526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369880:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.937853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519027147369876:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7265
2025-11-11T17:20:48.957997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519027147369876:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.973045Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519027147369861:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:48.978099Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519027147369798:2058] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:48.987122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519028427978861:2071] [node 4] ICP18 handshake (actor [1:7571519028427978920:2118]) is too old
2025-11-11T17:20:48.677007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519025123294013:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:48.936451Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519026869197109:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.775336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519021015036716:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.811416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519027255952535:2061] [node 5] ICS10 traffic start
2025-11-11T17:20:48.811440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:48.814653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519027255952531:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:48.831042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:48.870353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519022960985140:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(103360198ns) {HandleHandshakeStatus:329(103357161ns) {DropIncomingHandshake:441(785ns) DropOutgoingHandshake:456(5902ns) SwitchToState:178(605ns) ProcessPendingSessionEvents:495(13249475ns) {ForwardSessionEventToSession:612(13196567ns) ForwardSessionEventToSession:612(5171ns) ForwardSessionEventToSession:612(6783ns) ForwardSessionEventToSession:612(3016ns) ForwardSessionEventToSession:612(3670ns) ForwardSessionEventToSession:612(7928ns) ForwardSessionEventToSession:612(3179ns) ForwardSessionEventToSession:612(3020ns) ForwardSessionEventToSession:612(5042ns) ForwardSessionEventToSession:612(3104ns)}}}
2025-11-11T17:20:48.877708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519027255952459:2063] [node 6] ICP35 event processing took too much time PendingConnection:278(190055685ns) {HandleHandshakeStatus:329(190052887ns) {DropIncomingHandshake:441(110ns) DropOutgoingHandshake:456(506ns) SwitchToState:178(150ns) ProcessPendingSessionEvents:495(27772131ns) {ForwardSessionEventToSession:612(65950ns) ForwardSessionEventToSession:612(3308ns) ForwardSessionEventToSession:612(8094ns) ForwardSessionEventToSession:612(7943314ns) ForwardSessionEventToSession:612(9184ns) ForwardSessionEventToSession:612(13915469ns) ForwardSessionEventToSession:612(2789ns) ForwardSessionEventToSession:612(5791702ns) ForwardSessionEventToSession:612(6597ns) ForwardSessionEventToSession:612(3065ns)}}}
2025-11-11T17:20:48.998524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519022852402492:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.998633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:48.998709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519022852402492:2049] [node 2] ICP22 created new session: [4:7571519027147369882:2049]
2025-11-11T17:20:48.998762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519027147369882:2049] [node 2] ICS09 handshake done sender: [4:7571519027147369861:2114] self: [4:7571519027147369879:0] peer: [2:7571519026869197108:0] socket: 554 qp: -1
2025-11-11T17:20:48.998799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519027147369882:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:48.998853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:48.999351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519022852402491:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.999415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:48.999453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519022852402491:2048] [node 1] ICP22 created new session: [4:7571519027147369884:2048]
2025-11-11T17:20:48.999506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519027147369884:2048] [node 1] ICS09 handshake done sender: [4:7571519027147369798:2058] self: [4:7571519027147369877:0] peer: [1:7571519028427978868:0] socket: 521 qp: -1
2025-11-11T17:20:48.999538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519027147369884:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:48.999569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:49.002902Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519027147369873:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:49.002956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519027147369873:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:48.677046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:48.703422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519025123294009:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.727322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519025123294011:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(48053730ns) {Configure:104(48048051ns) {ProcessConfigured:135(59104ns) {StartInitialHandshake:157(56452ns) {DropHandshakes:469(3332ns) {DropIncomingHandshake:441(643ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(639ns) PrepareNewSessionHandshake:476(1638ns) {SwitchToState:178(530ns)}}}}}
2025-11-11T17:20:48.727582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519025123294011:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.727653Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519025123293947:2059] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:48.742635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519020828326641:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.742695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:48.742744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519020828326641:2050] [node 3] ICP22 created new session: [6:7571519025123294018:2050]
2025-11-11T17:20:48.742794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519025123294018:2050] [node 3] ICS09 handshake done sender: [6:7571519025123293947:2059] self: [6:7571519025123293948:0] peer: [3:7571519027255952460:0] socket: 520 qp: -1
2025-11-11T17:20:48.742831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519025123294018:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:48.742869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:48.745828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519025123294010:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.750281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519025123293958:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(164226236ns) {Configure:104(164220006ns) {ProcessConfigured:135(114636719ns) {StartInitialHandshake:157(114632535ns) {DropHandshakes:469(3708ns) {DropIncomingHandshake:441(991ns) DropOutgoingHandshake:456(604ns)} GenerateSessionVirtualId:988(1446ns) PrepareNewSessionHandshake:476(3982ns) {SwitchToState:178(1581ns)}}}}}
2025-11-11T17:20:48.822109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123294022:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.825879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519025123293958:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.861780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519025123293958:2066] [node 5] ICP35 event processing took too much time PendingConnection:278(32064068ns) {IncomingHandshake:261(8234235ns) {DropIncomingHandshake:441(502ns) PrepareNewSessionHandshake:476(1905ns) {SwitchToState:178(863ns)} GenerateSessionVirtualId:988(647ns) IssueIncomingHandshakeReply:193(8195002ns) {SwitchToState:178(2067ns)}}}
2025-11-11T17:20:48.882341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123294024:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.886137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519025123294025:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.889684Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519025123294009:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:48.897555Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:48.905913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519025123294011:2077] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519025123294009:2116] held: no
2025-11-11T17:20:48.946237Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519025123294010:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:48.950414Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519025123294025:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.950536Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:48.966899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519025123293958:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519025123294010:2117] held: no
2025-11-11T17:20:48.971942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519020828326639:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.972002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519025123293958:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.972041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:48.972089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519025123293958:2066] [node 5] ICP22 created new session: [6:7571519025123294027:2066]
2025-11-11T17:20:48.972128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519025123294027:2066] [node 5] ICS09 handshake done sender: [6:7571519025123294010:2117] self: [6:7571519025123294021:0] peer: [5:7571519025310004080:0] socket: 549 qp: -1
2025-11-11T17:20:48.972165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519025123294027:2066] [node 5] ICS10 traffic start
2025-11-11T17:20:48.972195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:48.978288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519025123294011:2077] [node 4] ICP35 event processing took too much time PendingConnection:278(18707710ns) {HandleHandshakeStatus:395(18703999ns) {LogHandshakeFail:472(21196ns) {UpdateErrorStateLog:544(6846ns)} DropOutgoingHandshake:456(2827ns) ProcessPendingSessionEvents:495(10655734ns) {DropSessionEvent:512(10103ns) DropSessionEvent:512(5802ns) DropSessionEvent:512(5101ns) DropSessionEvent:512(5486ns) DropSessionEvent:512(16835ns) DropSessionEvent:512(4641ns) DropSessionEvent:512(5505ns) DropSessionEvent:512(4169ns) DropSessionEvent:512(4989ns) DropSessionEvent:512(3989ns)}}}
2025-11-11T17:20:48.985861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519025123294011:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.985951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:48.986033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519025123294011:2077] [node 4] ICP22 created new session: [6:7571519025123294029:2077]
2025-11-11T17:20:48.986093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519025123294029:2077] [node 4] ICS09 handshake done sender: [6:7571519025123294009:2116] self: [6:7571519025123294017:0] peer: [4:7571519027147369826:0] socket: 548 qp: -1
2025-11-11T17:20:48.986132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519025123294029:2077] [node 4] ICS10 traffic start
2025-11-11T17:20:48.986197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:49.007890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519029418261327:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12921
2025-11-11T17:20:48.792036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519021015036717:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(40618230ns) {IncomingHandshake:261(4446191ns) {DropIncomingHandshake:441(2299ns) PrepareNewSessionHandshake:476(6119ns) {SwitchToState:178(4481ns)} GenerateSessionVirtualId:988(959ns) IssueIncomingHandshakeReply:193(88992ns) {SwitchToState:178(492ns)}}}
2025-11-11T17:20:48.994079Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519028427978922:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.936476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519026869197109:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.949784Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519026869197109:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:48.971281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519026869197070:2081] [node 5] ICP35 event processing took too much time PendingConnection:278(110529214ns) {HandleHandshakeStatus:329(110526904ns) {DropIncomingHandshake:441(725ns) DropOutgoingHandshake:456(538ns) SwitchToState:178(9865ns) ProcessPendingSessionEvents:495(22396137ns) {ForwardSessionEventToSession:612(22346300ns) ForwardSessionEventToSession:612(5385ns) ForwardSessionEventToSession:612(4783ns) ForwardSessionEventToSession:612(2956ns) ForwardSessionEventToSession:612(2547ns) ForwardSessionEventToSession:612(5004ns) ForwardSessionEventToSession:612(1840ns) ForwardSessionEventToSession:612(2194ns) ForwardSessionEventToSession:612(7877ns) ForwardSessionEventToSession:612(2709ns)}}}
2025-11-11T17:20:48.990291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519026869197047:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.990353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:48.990400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519026869197047:2063] [node 4] ICP22 created new session: [2:7571519026869197122:2063]
2025-11-11T17:20:48.990446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519026869197122:2063] [node 4] ICS09 handshake done sender: [2:7571519026869197109:2116] self: [2:7571519026869197108:0] peer: [4:7571519027147369879:0] socket: 553 qp: -1
2025-11-11T17:20:48.990478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519026869197122:2063] [node 4] ICS10 traffic start
2025-11-11T17:20:48.990505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:49.027395Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519025123294025:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:49.028070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519020828326639:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:49.037782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:49.037849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519020828326639:2048] [node 1] ICP22 created new session: [6:7571519029418261328:2048]
2025-11-11T17:20:49.037898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519029418261328:2048] [node 1] ICS09 handshake done sender: [6:7571519025123294025:2121] self: [6:7571519025123294026:0] peer: [1:7571519028427978919:0] socket: 575 qp: -1
2025-11-11T17:20:49.037947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519029418261328:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:49.038027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:49.026704Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519027147369873:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:48.994121Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519028427978922:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:49.015227Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519028427978922:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:49.046049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519028427978886:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.805953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519021015036716:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(25592652ns) {IncomingHandshake:261(25588766ns) {DropIncomingHandshake:441(884ns) PrepareNewSessionHandshake:476(975411ns) {SwitchToState:178(3434ns)} GenerateSessionVirtualId:988(1527ns) IssueIncomingHandshakeReply:193(77301ns) {SwitchToState:178(914ns)}}}
2025-11-11T17:20:49.046126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:49.046180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519028427978886:2084] [node 6] ICP22 created new session: [1:7571519032722946226:2084]
2025-11-11T17:20:49.046228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519032722946226:2084] [node 6] ICS09 handshake done sender: [1:7571519028427978922:2119] self: [1:7571519028427978919:0] peer: [6:7571519025123294026:0] socket: 574 qp: -1
2025-11-11T17:20:49.046263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519032722946226:2084] [node 6] ICS10 traffic start
2025-11-11T17:20:49.046300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:48.812621Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22830) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:48.814207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004095:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.817662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519021015036715:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.817904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519021015036717:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22830) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519025310004079:2111] held: no
2025-11-11T17:20:48.818064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.833919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519021015036715:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(39966304ns) {IncomingHandshake:261(20059032ns) {DropIncomingHandshake:441(569ns) PrepareNewSessionHandshake:476(2439ns) {SwitchToState:178(1437ns)} GenerateSessionVirtualId:988(1166ns) IssueIncomingHandshakeReply:193(20006079ns) {SwitchToState:178(3452ns)}}}
2025-11-11T17:20:48.878956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519025310004081:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.878979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519025310004081:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:48.880758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004097:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.881031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004098:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.881261Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519025310004079:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:48.894246Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519025310004081:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:48.894649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519021015036717:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.894706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:48.894752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519021015036717:2050] [node 3] ICP22 created new session: [5:7571519025310004102:2050]
2025-11-11T17:20:48.894819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519025310004102:2050] [node 3] ICS09 handshake done sender: [5:7571519025310004079:2111] self: [5:7571519025310004092:0] peer: [3:7571519027255952483:0] socket: 541 qp: -1
2025-11-11T17:20:48.894856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519025310004102:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:48.894890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:48.895329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519025310004038:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:48.895363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:48.895406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519025310004038:2062] [node 6] ICP22 created new session: [5:7571519025310004104:2062]
2025-11-11T17:20:48.895448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519025310004104:2062] [node 6] ICS09 handshake done sender: [5:7571519025310004081:2112] self: [5:7571519025310004080:0] peer: [6:7571519025123294021:0] socket: 546 qp: -1
2025-11-11T17:20:48.895498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519025310004104:2062] [node 6] ICS10 traffic start
2025-11-11T17:20:48.895534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:48.895947Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519025310004032:2066] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:48.913874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519025310004031:2065] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:48.914047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519025310004099:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:48.915673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519021015036716:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.915732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:48.915784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519021015036716:2049] [node 2] ICP22 created new session: [5:7571519025310004106:2049]
2025-11-11T17:20:48.915866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519025310004106:2049] [node 2] ICS09 handshake done sender: [5:7571519025310004032:2066] self: [5:7571519025310004093:0] peer: [2:7571519026869197071:0] socket: 537 qp: -1
2025-11-11T17:20:48.915906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519025310004106:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:48.915944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:48.926109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519021015036715:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:48.926187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:48.926235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519021015036715:2048] [node 1] ICP22 created new session: [5:7571519025310004109:2048]
2025-11-11T17:20:48.926279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519025310004109:2048] [node 1] ICS09 handshake done sender: [5:7571519025310004031:2065] self: [5:7571519025310004094:0] peer: [1:7571519028427978872:0] socket: 534 qp: -1
2025-11-11T17:20:48.926316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519025310004109:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:48.927981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:48.942571Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519025310004098:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.942810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571519025310004038:2062] [node 6] ICP18 handshake (actor [5:7571519025310004098:2121]) is too old
2025-11-11T17:20:48.961833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519025310004099:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:48.962008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519025310004033:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:48.991671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519021015036715:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(65251981ns) {HandleHandshakeStatus:329(65249738ns) {DropIncomingHandshake:441(789ns) DropOutgoingHandshake:456(4049ns) SwitchToState:178(620ns) ProcessPendingSessionEvents:495(17572993ns) {ForwardSessionEventToSession:612(17508042ns) ForwardSessionEventToSession:612(10180ns) ForwardSessionEventToSession:612(3560ns) ForwardSessionEventToSession:612(4155ns) ForwardSessionEventToSession:612(5536ns) ForwardSessionEventToSession:612(5304ns) ForwardSessionEventToSession:612(3931ns) ForwardSessionEventToSession:612(5485ns) ForwardSessionEventToSession:612(3305ns) ForwardSessionEventToSession:612(8992ns)}}}
2025-11-11T17:20:49.058175Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:49.061730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519025310004033:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519025310004099:2122] held: no
2025-11-11T17:20:49.027732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519027147369853:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:49.086975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519028427978886:2084] [node 6] ICP35 event processing took too much time PendingConnection:278(40666668ns) {HandleHandshakeStatus:329(40664060ns) {DropIncomingHandshake:441(23567ns) DropOutgoingHandshake:456(380ns) SwitchToState:178(865ns) ProcessPendingSessionEvents:495(40265595ns) {ForwardSessionEventToSession:612(12521411ns) ForwardSessionEventToSession:612(5821ns) ForwardSessionEventToSession:612(3278ns) ForwardSessionEventToSession:612(19074937ns) ForwardSessionEventToSession:612(14332ns) ForwardSessionEventToSession:612(7598873ns) ForwardSessionEventToSession:612(9779ns) ForwardSessionEventToSession:612(2959ns) ForwardSessionEventToSession:612(1006434ns) ForwardSessionEventToSession:612(2745ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:49.027822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:49.027869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519027147369853:2070] [node 5] ICP22 created new session: [4:7571519031442337183:2070]
2025-11-11T17:20:49.046577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519031442337183:2070] [node 5] ICS09 handshake done sender: [4:7571519027147369873:2118] self: [4:7571519027147369872:0] peer: [5:7571519025310004111:0] socket: 567 qp: -1
2025-11-11T17:20:49.058716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519031442337183:2070] [node 5] ICS10 traffic start
2025-11-11T17:20:49.062480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:49.086273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519027147369853:2070] [node 5] ICP35 event processing took too much time PendingConnection:278(58272860ns) {HandleHandshakeStatus:329(58269986ns) {DropIncomingHandshake:441(15028ns) DropOutgoingHandshake:456(522ns) SwitchToState:178(457ns) ProcessPendingSessionEvents:495(7863016ns) {ForwardSessionEventToSession:612(7801094ns) ForwardSessionEventToSession:612(9004ns) ForwardSessionEventToSession:612(12831ns) ForwardSessionEventToSession:612(6087ns) ForwardSessionEventToSession:612(4281ns) ForwardSessionEventToSession:612(3666ns) ForwardSessionEventToSession:612(6070ns) ForwardSessionEventToSession:612(3023ns) ForwardSessionEventToSession:612(2634ns) ForwardSessionEventToSession:612(5339ns)}}}
2025-11-11T17:20:49.087885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519029604971408:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51997
2025-11-11T17:20:49.088894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519025310004099:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:49.089131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519025310004033:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:49.089200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:49.089248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519025310004033:2064] [node 4] ICP22 created new session: [5:7571519029604971409:2064]
2025-11-11T17:20:49.089290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519029604971409:2064] [node 4] ICS09 handshake done sender: [5:7571519025310004099:2122] self: [5:7571519025310004111:0] peer: [4:7571519027147369872:0] socket: 569 qp: -1
2025-11-11T17:20:49.089324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519029604971409:2064] [node 4] ICS10 traffic start
2025-11-11T17:20:49.089365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:49.157717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:49.157717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:49.157951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519026869197111:2048] [node 1] ICS01 socket: 519 reason# EndOfStream
2025-11-11T17:20:49.157993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519022574229727:2048] [node 1] ICP30 unregister session Session# [2:7571519026869197111:2048] VirtualId# [2:7571519026869197057:0]
2025-11-11T17:20:49.158026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519026869197111:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.158122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:49.158171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:49.158060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519027255952485:2048] [node 1] ICS07 socket disconnect 512 reason# EndOfStream
2025-11-11T17:20:49.158094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519027255952485:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.158170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:49.158209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519027255952485:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:49.162859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:49.163065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519025310004109:2048] [node 1] ICS07 socket disconnect 534 reason# EndOfStream
2025-11-11T17:20:49.163093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519025310004109:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.163190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:49.163248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519025310004109:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:49.162824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:49.163070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519027147369884:2048] [node 1] ICS01 socket: 521 reason# EndOfStream
2025-11-11T17:20:49.163114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519022852402491:2048] [node 1] ICP30 unregister session Session# [4:7571519027147369884:2048] VirtualId# [4:7571519027147369877:0]
2025-11-11T17:20:49.163150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519027147369884:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.163241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:49.163286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:49.167206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:49.175746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519029418261328:2048] [node 1] ICS01 socket: 575 reason# EndOfStream
2025-11-11T17:20:49.175794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519020828326639:2048] [node 1] ICP30 unregister session Session# [6:7571519029418261328:2048] VirtualId# [6:7571519025123294026:0]
2025-11-11T17:20:49.175837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519029418261328:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.175963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:49.176023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:49.176217Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address
2025-11-11T17:20:49.176544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519022960985139:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:49.197763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519027255952485:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:49.197828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519022960985139:2048] [node 1] ICP30 unregister session Session# [3:7571519027255952485:2048] VirtualId# [3:7571519027255952469:0]
2025-11-11T17:20:49.201663Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address
2025-11-11T17:20:49.204165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:49.221801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519022960985139:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.221828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.222131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [3:7571519022960985139:2048] [node 1] ICP35 event processing took too much time StateWork:293(45350391ns) {HandleHandshakeStatus:395(45347573ns) {LogHandshakeFail:472(7272ns) {UpdateErrorStateLog:544(2183ns)} DropOutgoingHandshake:456(498ns) ProcessPendingSessionEvents:495(356ns) UnregisterSession:538(50581ns) {ProcessPendingSessionEvents:495(313ns) SwitchToInitialState:201(1646ns) {SwitchToState:178(561ns)}} TransitToErrorState:752(104635ns) {SwitchToState:178(6452ns) ProcessPendingSessionEvents:495(730ns) DropHandshakes:469(2568ns) {DropIncomingHandshake:441(875ns) DropOutgoingHandshake:456(296ns)}}}}
2025-11-11T17:20:49.233762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519022960985139:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:49.202081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519021015036715:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:49.202152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519025310004109:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:49.202194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519021015036715:2048] [node 1] ICP30 unregister session Session# [5:7571519025310004109:2048] VirtualId# [5:7571519025310004094:0]
2025-11-11T17:20:49.202253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:49.202289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519021015036715:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.202306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.258537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519021015036715:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:49.273951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:49.274167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519027255952531:2049] [node 2] ICS07 socket disconnect 516 reason# EndOfStream
2025-11-11T17:20:49.274196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519027255952531:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.274270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:49.274306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519027255952531:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:49.277315Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5968) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:20:49.277702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519022960985140:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5968) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:20:49.277824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519027255952531:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:49.278389Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5968) Couldn't connect to any resolved address
2025-11-11T17:20:49.278629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519022960985140:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5968) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:49.278671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519027255952531:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:49.278705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519022960985140:2049] [node 2] ICP30 unregister session Session# [3:7571519027255952531:2049] VirtualId# [3:7571519027255952492:0]
2025-11-11T17:20:49.278752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:49.278782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519022960985140:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.278795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.285809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:49.286050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519025123294013:2049] [node 2] ICS01 socket: 514 reason# EndOfStream
2025-11-11T17:20:49.286101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519020828326640:2049] [node 2] ICP30 unregister session Session# [6:7571519025123294013:2049] VirtualId# [6:7571519025123293949:0]
2025-11-11T17:20:49.286140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519025123294013:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.286239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:49.286289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:49.286414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:49.286629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519027147369882:2049] [node 2] ICS01 socket: 554 reason# EndOfStream
2025-11-11T17:20:49.286671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519022852402492:2049] [node 2] ICP30 unregister session Session# [4:7571519027147369882:2049] VirtualId# [4:7571519027147369879:0]
2025-11-11T17:20:49.286698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519027147369882:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.286752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:49.286788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:49.290554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:49.290773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519025310004106:2049] [node 2] ICS01 socket: 537 reason# EndOfStream
2025-11-11T17:20:49.290811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519021015036716:2049] [node 2] ICP30 unregister session Session# [5:7571519025310004106:2049] VirtualId# [5:7571519025310004093:0]
2025-11-11T17:20:49.290849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519025310004106:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.290935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:49.290980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:49.292008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519022960985140:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:49.477790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:49.478009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519027147369864:2050] [node 3] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-11T17:20:49.478043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519027147369864:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.478123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:49.478164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519027147369864:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:49.479050Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22830) Couldn't connect to any resolved address
2025-11-11T17:20:49.479286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519022852402493:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22830) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:49.479333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519027147369864:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:49.479363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519022852402493:2050] [node 3] ICP30 unregister session Session# [4:7571519027147369864:2050] VirtualId# [4:7571519027147369860:0]
2025-11-11T17:20:49.479407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:49.479436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519022852402493:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22830) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.479451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22830) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:49.481001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:49.481237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519025123294018:2050] [node 3] ICS01 socket: 520 reason# EndOfStream
2025-11-11T17:20:49.481303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519020828326641:2050] [node 3] ICP30 unregister session Session# [6:7571519025123294018:2050] VirtualId# [6:7571519025123293948:0]
2025-11-11T17:20:49.481356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519025123294018:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.481491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:49.481566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:49.481682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:49.481873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519025310004102:2050] [node 3] ICS01 socket: 541 reason# EndOfStream
2025-11-11T17:20:49.481934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519021015036717:2050] [node 3] ICP30 unregister session Session# [5:7571519025310004102:2050] VirtualId# [5:7571519025310004092:0]
2025-11-11T17:20:49.481985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519025310004102:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.482095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:49.482159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:49.553779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:49.553990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519029604971409:2064] [node 4] ICS01 socket: 569 reason# EndOfStream
2025-11-11T17:20:49.554035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519025310004033:2064] [node 4] ICP30 unregister session Session# [5:7571519029604971409:2064] VirtualId# [5:7571519025310004111:0]
2025-11-11T17:20:49.554070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519029604971409:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.554155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:49.554198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:49.554286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:49.554450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519025123294029:2077] [node 4] ICS01 socket: 548 reason# EndOfStream
2025-11-11T17:20:49.554483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519025123294011:2077] [node 4] ICP30 unregister session Session# [6:7571519025123294029:2077] VirtualId# [6:7571519025123294017:0]
2025-11-11T17:20:49.554506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519025123294029:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.554553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:49.554588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:49.650442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:49.650662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519025123294027:2066] [node 5] ICS01 socket: 549 reason# EndOfStream
2025-11-11T17:20:49.650707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519025123293958:2066] [node 5] ICP30 unregister session Session# [6:7571519025123294027:2066] VirtualId# [6:7571519025123294021:0]
2025-11-11T17:20:49.650742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519025123294027:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:49.650842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:49.650890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:49.843862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519031133332424:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:49.843910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519031133332425:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:49.885541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519030185742347:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:49.885603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519030185742348:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:49.917880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519031707227459:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:49.917932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519031707227460:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:49.985941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519033407210279:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:49.985984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519033407210280:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:49.986025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519033407210281:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:50.013915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519029404784489:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:50.013963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519029404784490:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:50.013990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519029404784491:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:50.282010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519036328391066:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:50.282061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519036328391067:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:50.282089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519036328391068:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:50.282722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519036328391079:2059] [node 4] ICP01 ready to work
2025-11-11T17:20:50.278467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519037702177603:2061] [node 5] ICP01 ready to work
2025-11-11T17:20:50.363742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519036328391068:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23680119ns) {Configure:104(23673236ns) {ProcessConfigured:135(100558ns) {StartInitialHandshake:157(96639ns) {DropHandshakes:469(2768ns) {DropIncomingHandshake:441(634ns) DropOutgoingHandshake:456(384ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:476(2554ns) {SwitchToState:178(1279ns)}}}}}
2025-11-11T17:20:50.345840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519036002194790:2066] [node 4] ICP01 ready to work
2025-11-11T17:20:50.320864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519035428299782:2072] [node 6] ICP01 ready to work
2025-11-11T17:20:50.423598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519033407210280:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(142833107ns) {Configure:104(142823191ns) {ProcessConfigured:135(55457801ns) {StartInitialHandshake:157(55453561ns) {DropHandshakes:469(26172534ns) {DropIncomingHandshake:441(1915ns) DropOutgoingHandshake:456(481ns)} GenerateSessionVirtualId:988(3819ns) PrepareNewSessionHandshake:476(6223ns) {SwitchToState:178(3194ns)}}}}}
2025-11-11T17:20:50.424057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519037702177638:2070] [node 6] ICP01 ready to work
2025-11-11T17:20:50.356312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519034480709706:2072] [node 6] ICP01 ready to work
2025-11-11T17:20:50.461935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519036002194819:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:50.430119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519035428299784:2073] [node 4] ICP01 ready to work
2025-11-11T17:20:50.474475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519034480709707:2073] [node 5] ICP01 ready to work
2025-11-11T17:20:50.474814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519034480709708:2074] [node 4] ICP01 ready to work
2025-11-11T17:20:50.470188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519036002194820:2071] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.495242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519033407210281:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(66290306ns) {Configure:104(66286732ns) {ProcessConfigured:135(56184ns) {StartInitialHandshake:157(53216ns) {DropHandshakes:469(951ns) {DropIncomingHandshake:441(215ns) DropOutgoingHandshake:456(149ns)} GenerateSessionVirtualId:988(417ns) PrepareNewSessionHandshake:476(1068ns) {SwitchToState:178(443ns)}}}}}
2025-11-11T17:20:50.514431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519037702177651:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2827
2025-11-11T17:20:50.493763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519035428299785:2074] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.522655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519034480709722:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48435
2025-11-11T17:20:50.530056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519036328391066:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(107469638ns) {Configure:104(107464173ns) {ProcessConfigured:135(1970670ns) {StartInitialHandshake:157(1967990ns) {DropHandshakes:469(1766ns) {DropIncomingHandshake:441(432ns) DropOutgoingHandshake:456(274ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:476(4295ns) {SwitchToState:178(2065ns)}}}}}
2025-11-11T17:20:50.535385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519031707227459:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33375602ns) {Configure:104(33371554ns) {ProcessConfigured:135(10916451ns) {StartInitialHandshake:157(10914438ns) {DropHandshakes:469(1249ns) {DropIncomingHandshake:441(372ns) DropOutgoingHandshake:456(243ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:476(2836ns) {SwitchToState:178(278ns)}}}}}
2025-11-11T17:20:50.551069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519036328391138:2067] [node 5] ICP01 ready to work
2025-11-11T17:20:50.555850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519029404784489:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(373195441ns) {Configure:104(373186326ns) {ProcessConfigured:135(1281428ns) {StartInitialHandshake:157(1279211ns) {DropHandshakes:469(2078ns) {DropIncomingHandshake:441(491ns) DropOutgoingHandshake:456(494ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:476(3691ns) {SwitchToState:178(1601ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.576454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519036328391067:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22644287ns) {Configure:104(22614512ns) {ProcessConfigured:135(67899ns) {StartInitialHandshake:157(65425ns) {DropHandshakes:469(2511ns) {DropIncomingHandshake:441(523ns) DropOutgoingHandshake:456(429ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:476(3593ns) {SwitchToState:178(390ns)}}}}}
2025-11-11T17:20:50.578440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391139:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.556361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519033699751850:2061] [node 6] ICP01 ready to work
2025-11-11T17:20:50.556650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519033699751851:2073] [node 4] ICP01 ready to work
2025-11-11T17:20:50.602925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519035428299785:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(53281209ns) {Configure:104(53276807ns) {ProcessConfigured:135(66442ns) {StartInitialHandshake:157(63709ns) {DropHandshakes:469(3020ns) {DropIncomingHandshake:441(435ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(528ns) PrepareNewSessionHandshake:476(2270ns) {SwitchToState:178(1004ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.617878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519029404784490:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(40639693ns) {Configure:104(40634567ns) {ProcessConfigured:135(9331554ns) {StartInitialHandshake:157(9328737ns) {DropHandshakes:469(2582ns) {DropIncomingHandshake:441(429ns) DropOutgoingHandshake:456(634ns)} GenerateSessionVirtualId:988(1019ns) PrepareNewSessionHandshake:476(4892ns) {SwitchToState:178(2583ns)}}}}}
2025-11-11T17:20:50.618458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751852:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
2025-11-11T17:20:50.618641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751853:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.622351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391145:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.630780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519035428299792:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26144
2025-11-11T17:20:50.626712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519036328391139:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.626850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519036328391067:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.632270Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519033699751853:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.630963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519035428299793:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26144
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.635028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519031707227460:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(66632237ns) {Configure:104(66626644ns) {ProcessConfigured:135(64135ns) {StartInitialHandshake:157(61149ns) {DropHandshakes:469(2522ns) {DropIncomingHandshake:441(533ns) DropOutgoingHandshake:456(405ns)} GenerateSessionVirtualId:988(770ns) PrepareNewSessionHandshake:476(1457ns) {SwitchToState:178(584ns)}}}}}
2025-11-11T17:20:50.631072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519035428299794:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26144
2025-11-11T17:20:50.650124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519037702177655:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2827
2025-11-11T17:20:50.677676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519037702177651:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.661977Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709712:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.662055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519034480709712:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.631256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519035428299799:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26144
2025-11-11T17:20:50.635877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299794:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.665955Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299792:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.707590Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519037702177655:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.707809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519033407210279:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.708054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299799:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.706618Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519033699751852:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.706827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519029404784490:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.709845Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519037702177641:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.709877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519037702177641:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.711084Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519037702177641:2108] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.713924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391147:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.711624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519037702177603:2061] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.709152Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299789:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.716112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519036328391145:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.706953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751865:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
2025-11-11T17:20:50.707217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519033699751851:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.713336Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519033699751865:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.713450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519029404784489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.724712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751869:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
2025-11-11T17:20:50.711698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:50.711750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519037702177603:2061] [node 5] ICP22 created new session: [4:7571519037702177660:2061]
2025-11-11T17:20:50.711797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519037702177660:2061] [node 5] ICS09 handshake done sender: [4:7571519037702177641:2108] self: [4:7571519037702177640:0] peer: [5:7571519033699751867:0] socket: 521 qp: -1
2025-11-11T17:20:50.711842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519037702177660:2061] [node 5] ICS10 traffic start
2025-11-11T17:20:50.711883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:50.712524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519037702177659:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2827
2025-11-11T17:20:50.717797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519037702177655:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:50.722710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519033407210279:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.722771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:50.722826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519033407210279:2048] [node 1] ICP22 created new session: [4:7571519037702177663:2048]
2025-11-11T17:20:50.722873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519037702177663:2048] [node 1] ICS09 handshake done sender: [4:7571519037702177655:2117] self: [4:7571519037702177657:0] peer: [1:7571519035428299788:0] socket: 542 qp: -1
2025-11-11T17:20:50.722913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519037702177663:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:50.722978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.726893Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709714:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.726937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519034480709714:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.727893Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709722:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.728265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519034480709723:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.728841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519036328391138:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.729313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391149:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.725428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519036002194820:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(80721459ns) {Configure:104(80715047ns) {ProcessConfigured:135(56352ns) {StartInitialHandshake:157(53011ns) {DropHandshakes:469(2001ns) {DropIncomingHandshake:441(338ns) DropOutgoingHandshake:456(399ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:476(1735ns) {SwitchToState:178(623ns)}}}}}
2025-11-11T17:20:50.709188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519035428299789:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.726646Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519033699751852:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:50.714122Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519035428299789:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:50.729963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299791:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.729989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519035428299791:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.730701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299793:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.730745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519035428299784:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.730853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:50.730901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519035428299784:2073] [node 4] ICP22 created new session: [1:7571519035428299807:2073]
2025-11-11T17:20:50.730958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519035428299807:2073] [node 4] ICS09 handshake done sender: [1:7571519035428299789:2111] self: [1:7571519035428299788:0] peer: [4:7571519037702177657:0] socket: 539 qp: -1
2025-11-11T17:20:50.730991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519035428299807:2073] [node 4] ICS10 traffic start
2025-11-11T17:20:50.731048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:50.733222Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519033699751857:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.733259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519033699751857:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.737930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391150:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.738239Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519034480709712:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:50.738929Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519036328391139:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:50.739222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519036328391067:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.739320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:50.739366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519036328391067:2049] [node 2] ICP22 created new session: [6:7571519036328391151:2049]
2025-11-11T17:20:50.739407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519036328391151:2049] [node 2] ICS09 handshake done sender: [6:7571519036328391139:2113] self: [6:7571519036328391146:0] peer: [2:7571519034480709711:0] socket: 526 qp: -1
2025-11-11T17:20:50.739440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519036328391151:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:50.739487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:50.743082Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519034480709714:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:50.735653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519033699751851:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.735760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:50.735808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519033699751851:2073] [node 4] ICP22 created new session: [5:7571519033699751871:2073]
2025-11-11T17:20:50.735854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519033699751871:2073] [node 4] ICS09 handshake done sender: [5:7571519033699751852:2111] self: [5:7571519033699751867:0] peer: [4:7571519037702177640:0] socket: 527 qp: -1
2025-11-11T17:20:50.735891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519033699751871:2073] [node 4] ICS10 traffic start
2025-11-11T17:20:50.735950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:50.737235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751870:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.743827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709723:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.743966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519034480709706:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.744039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:50.744090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519034480709706:2072] [node 6] ICP22 created new session: [2:7571519034480709728:2072]
2025-11-11T17:20:50.744147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519034480709728:2072] [node 6] ICS09 handshake done sender: [2:7571519034480709712:2111] self: [2:7571519034480709711:0] peer: [6:7571519036328391146:0] socket: 524 qp: -1
2025-11-11T17:20:50.744183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519034480709728:2072] [node 6] ICS10 traffic start
2025-11-11T17:20:50.744236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:50.744817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519034480709707:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.744878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:50.744918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519034480709707:2073] [node 5] ICP22 created new session: [2:7571519034480709730:2073]
2025-11-11T17:20:50.744965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519034480709730:2073] [node 5] ICS09 handshake done sender: [2:7571519034480709714:2112] self: [2:7571519034480709713:0] peer: [5:7571519033699751866:0] socket: 525 qp: -1
2025-11-11T17:20:50.744989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519034480709730:2073] [node 5] ICS10 traffic start
2025-11-11T17:20:50.745014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:50.745804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519034480709727:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48435
2025-11-11T17:20:50.745923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519030185742347:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.747814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519036002194829:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31276
2025-11-11T17:20:50.747977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519036002194830:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31276
2025-11-11T17:20:50.748136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519036002194831:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31276
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.749510Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299787:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.749545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519035428299787:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.750654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519035428299810:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26144
2025-11-11T17:20:50.751040Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519036328391147:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.752860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519036328391079:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.753316Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519033699751857:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.754283Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194829:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.754691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519036002194837:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31276
2025-11-11T17:20:50.754741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519036002194838:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31276
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.758381Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519037702177643:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.758419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519037702177643:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.759433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519037702177666:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2827
2025-11-11T17:20:50.759531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519037702177667:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2827
2025-11-11T17:20:50.760383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519033699751850:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.760489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:50.760541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519033699751850:2061] [node 6] ICP22 created new session: [5:7571519033699751877:2061]
2025-11-11T17:20:50.760581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519033699751877:2061] [node 6] ICS09 handshake done sender: [5:7571519033699751857:2114] self: [5:7571519033699751856:0] peer: [6:7571519036328391148:0] socket: 535 qp: -1
2025-11-11T17:20:50.760615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519033699751877:2061] [node 6] ICS10 traffic start
2025-11-11T17:20:50.760676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:50.762865Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519033699751853:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:50.763378Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519035428299791:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:50.763865Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519035428299787:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:50.764137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519035428299785:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.764236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:50.764290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519035428299785:2074] [node 5] ICP22 created new session: [1:7571519035428299814:2074]
2025-11-11T17:20:50.764338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519035428299814:2074] [node 5] ICS09 handshake done sender: [1:7571519035428299791:2112] self: [1:7571519035428299790:0] peer: [5:7571519033699751868:0] socket: 540 qp: -1
2025-11-11T17:20:50.764376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519035428299814:2074] [node 5] ICS10 traffic start
2025-11-11T17:20:50.764423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:50.764833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519031133332424:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.764918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:50.764958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519031133332424:2048] [node 2] ICP22 created new session: [1:7571519035428299816:2048]
2025-11-11T17:20:50.764997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519035428299816:2048] [node 2] ICS09 handshake done sender: [1:7571519035428299787:2110] self: [1:7571519035428299786:0] peer: [2:7571519034480709734:0] socket: 538 qp: -1
2025-11-11T17:20:50.765033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519035428299816:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:50.765062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:50.766662Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519036328391145:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.766943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519036328391138:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.767017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:50.767063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519036328391138:2067] [node 5] ICP22 created new session: [6:7571519036328391156:2067]
2025-11-11T17:20:50.767109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519036328391156:2067] [node 5] ICS09 handshake done sender: [6:7571519036328391145:2116] self: [6:7571519036328391148:0] peer: [5:7571519033699751856:0] socket: 536 qp: -1
2025-11-11T17:20:50.767139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519036328391156:2067] [node 5] ICS10 traffic start
2025-11-11T17:20:50.767167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:50.767725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391155:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.768669Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519037702177643:2109] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:50.768980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519037702177638:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.769037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:50.769079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519037702177638:2070] [node 6] ICP22 created new session: [4:7571519037702177669:2070]
2025-11-11T17:20:50.769120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519037702177669:2070] [node 6] ICS09 handshake done sender: [4:7571519037702177643:2109] self: [4:7571519037702177642:0] peer: [6:7571519036328391154:0] socket: 546 qp: -1
2025-11-11T17:20:50.769152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519037702177669:2070] [node 6] ICS10 traffic start
2025-11-11T17:20:50.769198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:50.779419Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194831:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.779797Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194830:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.782106Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519037702177667:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.782256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519033407210281:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.783345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751875:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
2025-11-11T17:20:50.783542Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.786951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519034480709735:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48435
2025-11-11T17:20:50.787063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519034480709736:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48435
2025-11-11T17:20:50.789795Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194822:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.789473Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2827) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:50.789830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519036002194822:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.797797Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:50.798834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519030185742347:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519034480709723:2116] held: no
2025-11-11T17:20:50.798996Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519034480709723:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:50.799332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519030185742347:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.799418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:50.799493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519030185742347:2048] [node 1] ICP22 created new session: [2:7571519034480709737:2048]
2025-11-11T17:20:50.799538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519034480709737:2048] [node 1] ICS09 handshake done sender: [2:7571519034480709723:2116] self: [2:7571519034480709734:0] peer: [1:7571519035428299786:0] socket: 549 qp: -1
2025-11-11T17:20:50.799576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519034480709737:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:50.799630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:50.793863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519036328391079:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2827) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519036328391147:2117] held: no
2025-11-11T17:20:50.802623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751876:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
2025-11-11T17:20:50.794025Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519036328391147:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.810368Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:50.813038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519029404784490:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.813125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:50.803631Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519037702177666:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.804120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519036328391079:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.813321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519029404784490:2049] [node 2] ICP22 created new session: [5:7571519033699751879:2049]
2025-11-11T17:20:50.813972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519033407210280:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.813391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519033699751879:2049] [node 2] ICS09 handshake done sender: [5:7571519033699751853:2112] self: [5:7571519033699751866:0] peer: [2:7571519034480709713:0] socket: 528 qp: -1
2025-11-11T17:20:50.804211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:50.824821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519037702177673:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2827
2025-11-11T17:20:50.824898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519037702177674:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2827
2025-11-11T17:20:50.830195Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519036002194822:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:50.804281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519036328391079:2059] [node 4] ICP22 created new session: [6:7571519036328391159:2059]
2025-11-11T17:20:50.804337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519036328391159:2059] [node 4] ICS09 handshake done sender: [6:7571519036328391147:2117] self: [6:7571519036328391154:0] peer: [4:7571519037702177642:0] socket: 547 qp: -1
2025-11-11T17:20:50.804377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519036328391159:2059] [node 4] ICS10 traffic start
2025-11-11T17:20:50.804436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:50.805095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391158:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.821198Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519036328391158:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.833961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519036328391066:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.813430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519033699751879:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:50.821815Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709727:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.821987Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709716:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.822010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519034480709716:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.822942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519034480709707:2073] [node 5] ICP18 handshake (actor [2:7571519034480709727:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.850028Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519037702177667:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:50.838044Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299810:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.856849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391162:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.838198Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299798:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.813889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:50.814919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519033699751850:2061] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:50.843068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519029404784489:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519033699751865:2117] held: no
2025-11-11T17:20:50.845861Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519033699751865:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:50.846319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519029404784489:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.846411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:50.846460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519029404784489:2048] [node 1] ICP22 created new session: [5:7571519033699751881:2048]
2025-11-11T17:20:50.846526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519033699751881:2048] [node 1] ICS09 handshake done sender: [5:7571519033699751865:2117] self: [5:7571519033699751868:0] peer: [1:7571519035428299790:0] socket: 541 qp: -1
2025-11-11T17:20:50.846569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519033699751881:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:50.846608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:50.857302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519033407210281:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.857399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:50.857457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519033407210281:2050] [node 3] ICP22 created new session: [4:7571519037702177675:2050]
2025-11-11T17:20:50.857511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519037702177675:2050] [node 3] ICS09 handshake done sender: [4:7571519037702177667:2120] self: [4:7571519037702177671:0] peer: [3:7571519036002194821:0] socket: 575 qp: -1
2025-11-11T17:20:50.857549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519037702177675:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:50.861265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:50.865953Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709735:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.866877Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519034480709716:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:50.838223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519035428299798:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.843094Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519035428299798:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:50.850510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519035428299782:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.850622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:50.850678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519035428299782:2072] [node 6] ICP22 created new session: [1:7571519035428299821:2072]
2025-11-11T17:20:50.850726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519035428299821:2072] [node 6] ICS09 handshake done sender: [1:7571519035428299798:2117] self: [1:7571519035428299797:0] peer: [6:7571519036328391161:0] socket: 516 qp: -1
2025-11-11T17:20:50.850768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519035428299821:2072] [node 6] ICS10 traffic start
2025-11-11T17:20:50.850813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:50.868586Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194838:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.868690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194837:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.868733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519036002194790:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.868836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:50.868891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519036002194790:2066] [node 4] ICP22 created new session: [3:7571519036002194843:2066]
2025-11-11T17:20:50.868946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519036002194843:2066] [node 4] ICS09 handshake done sender: [3:7571519036002194822:2110] self: [3:7571519036002194821:0] peer: [4:7571519037702177671:0] socket: 564 qp: -1
2025-11-11T17:20:50.868990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519036002194843:2066] [node 4] ICS10 traffic start
2025-11-11T17:20:50.869058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:50.870424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519034480709708:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.870861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:50.870924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519034480709708:2074] [node 4] ICP22 created new session: [2:7571519034480709740:2074]
2025-11-11T17:20:50.870965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519034480709740:2074] [node 4] ICS09 handshake done sender: [2:7571519034480709716:2113] self: [2:7571519034480709715:0] peer: [4:7571519037702177672:0] socket: 562 qp: -1
2025-11-11T17:20:50.870996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519034480709740:2074] [node 4] ICS10 traffic start
2025-11-11T17:20:50.871033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:50.870034Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:50.874912Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:50.876024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519033407210280:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519037702177666:2119] held: no
2025-11-11T17:20:50.876154Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519037702177666:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:50.876385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519033407210280:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.876449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:50.876503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519033407210280:2049] [node 2] ICP22 created new session: [4:7571519037702177677:2049]
2025-11-11T17:20:50.876547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519037702177677:2049] [node 2] ICS09 handshake done sender: [4:7571519037702177666:2119] self: [4:7571519037702177672:0] peer: [2:7571519034480709715:0] socket: 574 qp: -1
2025-11-11T17:20:50.876580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519037702177677:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:50.876620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:50.881131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519031707227459:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.881202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519031707227460:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.881959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519035428299796:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.881991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519035428299796:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.881990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519036328391066:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519036328391158:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.883096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751883:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
2025-11-11T17:20:50.893702Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519036328391158:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:50.906217Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194828:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.906257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519036002194828:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.907290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519036002194848:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31276
2025-11-11T17:20:50.909270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519036002194828:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.901910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519033699751883:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.897823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519034480709718:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.902054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519029404784491:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.917934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519036328391066:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.918041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:50.930183Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:50.897899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519034480709718:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:50.907607Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519034480709718:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:50.907865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519030185742348:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.907964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:50.908014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519030185742348:2049] [node 3] ICP22 created new session: [2:7571519034480709744:2049]
2025-11-11T17:20:50.908067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519034480709744:2049] [node 3] ICS09 handshake done sender: [2:7571519034480709718:2114] self: [2:7571519034480709717:0] peer: [3:7571519036002194847:0] socket: 563 qp: -1
2025-11-11T17:20:50.908111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519034480709744:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:50.908170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:50.918219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519036328391066:2048] [node 1] ICP22 created new session: [6:7571519036328391163:2048]
2025-11-11T17:20:50.922665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519036328391163:2048] [node 1] ICS09 handshake done sender: [6:7571519036328391158:2121] self: [6:7571519036328391161:0] peer: [1:7571519035428299797:0] socket: 522 qp: -1
2025-11-11T17:20:50.922713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519036328391163:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:50.922793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:50.930462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519033699751885:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1372
2025-11-11T17:20:50.929919Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519035428299796:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:50.938572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519031133332425:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.938699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:50.938754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519031133332425:2049] [node 3] ICP22 created new session: [1:7571519035428299824:2049]
2025-11-11T17:20:50.938798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519035428299824:2049] [node 3] ICS09 handshake done sender: [1:7571519035428299796:2116] self: [1:7571519035428299795:0] peer: [3:7571519036002194846:0] socket: 569 qp: -1
2025-11-11T17:20:50.938853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519035428299824:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:50.938891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:50.950284Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519033699751883:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:50.951055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519029404784491:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.951144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:50.951199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519029404784491:2050] [node 3] ICP22 created new session: [5:7571519033699751886:2050]
2025-11-11T17:20:50.951249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519033699751886:2050] [node 3] ICS09 handshake done sender: [5:7571519033699751883:2122] self: [5:7571519033699751884:0] peer: [3:7571519036002194827:0] socket: 520 qp: -1
2025-11-11T17:20:50.951285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519033699751886:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:50.951321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:50.953054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519036002194820:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.953165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:50.953216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519036002194820:2071] [node 5] ICP22 created new session: [3:7571519036002194850:2071]
2025-11-11T17:20:50.953268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519036002194850:2071] [node 5] ICS09 handshake done sender: [3:7571519036002194828:2113] self: [3:7571519036002194827:0] peer: [5:7571519033699751884:0] socket: 519 qp: -1
2025-11-11T17:20:50.953304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519036002194850:2071] [node 5] ICS10 traffic start
2025-11-11T17:20:50.953360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:50.953973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519031707227460:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519036002194837:2118] held: no
2025-11-11T17:20:50.954271Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:50.956604Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519036002194837:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:50.957047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519031707227459:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519036002194838:2119] held: no
2025-11-11T17:20:50.957296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519031707227460:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.957367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:50.957410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519031707227460:2049] [node 2] ICP22 created new session: [3:7571519036002194854:2049]
2025-11-11T17:20:50.957455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519036002194854:2049] [node 2] ICS09 handshake done sender: [3:7571519036002194837:2118] self: [3:7571519036002194847:0] peer: [2:7571519034480709717:0] socket: 567 qp: -1
2025-11-11T17:20:50.957502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519036002194854:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:50.957533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.960677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519036002194852:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31276
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.961147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391165:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.961773Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519036002194838:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:50.962747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519031707227459:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.962820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:50.962876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519031707227459:2048] [node 1] ICP22 created new session: [3:7571519036002194856:2048]
2025-11-11T17:20:50.962920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519036002194856:2048] [node 1] ICS09 handshake done sender: [3:7571519036002194838:2119] self: [3:7571519036002194846:0] peer: [1:7571519035428299795:0] socket: 573 qp: -1
2025-11-11T17:20:50.962959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519036002194856:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:50.963014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:50.982711Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519036328391165:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.982881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519036328391068:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:50.985826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519036002194833:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:50.985866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519036002194833:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:50.987397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519036328391167:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54798
2025-11-11T17:20:50.989866Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519036002194833:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:50.990521Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519036328391165:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:50.990832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519036328391068:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:50.990929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:50.990979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519036328391068:2050] [node 3] ICP22 created new session: [6:7571519036328391168:2050]
2025-11-11T17:20:50.991039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519036328391168:2050] [node 3] ICS09 handshake done sender: [6:7571519036328391165:2123] self: [6:7571519036328391166:0] peer: [3:7571519036002194832:0] socket: 554 qp: -1
2025-11-11T17:20:50.991076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519036328391168:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:50.991139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:50.993912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519036002194819:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:50.994025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:50.994076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519036002194819:2067] [node 6] ICP22 created new session: [3:7571519036002194859:2067]
2025-11-11T17:20:50.994122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519036002194859:2067] [node 6] ICS09 handshake done sender: [3:7571519036002194833:2117] self: [3:7571519036002194832:0] peer: [6:7571519036328391166:0] socket: 534 qp: -1
2025-11-11T17:20:50.994156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519036002194859:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:50.994206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:51.049772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:51.050012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519036002194856:2048] [node 1] ICS01 socket: 573 reason# EndOfStream
2025-11-11T17:20:51.050061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519031707227459:2048] [node 1] ICP30 unregister session Session# [3:7571519036002194856:2048] VirtualId# [3:7571519036002194846:0]
2025-11-11T17:20:51.050100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519036002194856:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.050204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:51.050257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:51.053740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:51.053983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519034480709737:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-11-11T17:20:51.054039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519030185742347:2048] [node 1] ICP30 unregister session Session# [2:7571519034480709737:2048] VirtualId# [2:7571519034480709734:0]
2025-11-11T17:20:51.054073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519034480709737:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.054161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:51.054208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:51.057763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:51.057989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519036328391163:2048] [node 1] ICS01 socket: 522 reason# EndOfStream
2025-11-11T17:20:51.058029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519036328391066:2048] [node 1] ICP30 unregister session Session# [6:7571519036328391163:2048] VirtualId# [6:7571519036328391161:0]
2025-11-11T17:20:51.058062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519036328391163:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.058140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:51.058180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:51.058264Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:51.058484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519037702177663:2048] [node 1] ICS01 socket: 542 reason# EndOfStream
2025-11-11T17:20:51.058515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519033407210279:2048] [node 1] ICP30 unregister session Session# [4:7571519037702177663:2048] VirtualId# [4:7571519037702177657:0]
2025-11-11T17:20:51.058538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519037702177663:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.058592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:51.058622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:51.058784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:51.058972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519033699751881:2048] [node 1] ICS01 socket: 541 reason# EndOfStream
2025-11-11T17:20:51.059011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519029404784489:2048] [node 1] ICP30 unregister session Session# [5:7571519033699751881:2048] VirtualId# [5:7571519033699751868:0]
2025-11-11T17:20:51.059037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519033699751881:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.059100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:51.059133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:51.173754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:51.174033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519036002194854:2049] [node 2] ICS01 socket: 567 reason# EndOfStream
2025-11-11T17:20:51.174083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519031707227460:2049] [node 2] ICP30 unregister session Session# [3:7571519036002194854:2049] VirtualId# [3:7571519036002194847:0]
2025-11-11T17:20:51.174118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519036002194854:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.174197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:51.174237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:51.189780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:51.189780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:51.189963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:51.190154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519037702177677:2049] [node 2] ICS01 socket: 574 reason# EndOfStream
2025-11-11T17:20:51.190189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519033407210280:2049] [node 2] ICP30 unregister session Session# [4:7571519037702177677:2049] VirtualId# [4:7571519037702177672:0]
2025-11-11T17:20:51.190217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519037702177677:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.190286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:51.190327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:51.190113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519036328391151:2049] [node 2] ICS01 socket: 526 reason# EndOfStream
2025-11-11T17:20:51.190161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519036328391067:2049] [node 2] ICP30 unregister session Session# [6:7571519036328391151:2049] VirtualId# [6:7571519036328391146:0]
2025-11-11T17:20:51.190193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519036328391151:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.190280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:51.190329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:51.190902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519033699751879:2049] [node 2] ICS01 socket: 528 reason# EndOfStream
2025-11-11T17:20:51.190948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519029404784490:2049] [node 2] ICP30 unregister session Session# [5:7571519033699751879:2049] VirtualId# [5:7571519033699751866:0]
2025-11-11T17:20:51.190974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519033699751879:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.191037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:51.191076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:51.241836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:51.242056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519037702177675:2050] [node 3] ICS01 socket: 575 reason# EndOfStream
2025-11-11T17:20:51.242102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519033407210281:2050] [node 3] ICP30 unregister session Session# [4:7571519037702177675:2050] VirtualId# [4:7571519037702177671:0]
2025-11-11T17:20:51.242138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519037702177675:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.242228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:51.242272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:51.242372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:51.242536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519036328391168:2050] [node 3] ICS01 socket: 554 reason# EndOfStream
2025-11-11T17:20:51.242578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519036328391068:2050] [node 3] ICP30 unregister session Session# [6:7571519036328391168:2050] VirtualId# [6:7571519036328391166:0]
2025-11-11T17:20:51.242604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519036328391168:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.242652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:51.242685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:51.242761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:51.242948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519033699751886:2050] [node 3] ICS01 socket: 520 reason# EndOfStream
2025-11-11T17:20:51.242985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519029404784491:2050] [node 3] ICP30 unregister session Session# [5:7571519033699751886:2050] VirtualId# [5:7571519033699751884:0]
2025-11-11T17:20:51.243007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519033699751886:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.243064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:51.243093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:51.323086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:51.325769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:51.323341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519036328391159:2059] [node 4] ICS01 socket: 547 reason# EndOfStream
2025-11-11T17:20:51.323406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519036328391079:2059] [node 4] ICP30 unregister session Session# [6:7571519036328391159:2059] VirtualId# [6:7571519036328391154:0]
2025-11-11T17:20:51.323461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519036328391159:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.323630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:51.323702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:51.326055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519033699751871:2073] [node 4] ICS01 socket: 527 reason# EndOfStream
2025-11-11T17:20:51.326110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519033699751851:2073] [node 4] ICP30 unregister session Session# [5:7571519033699751871:2073] VirtualId# [5:7571519033699751867:0]
2025-11-11T17:20:51.326161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519033699751871:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.326270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:51.326347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:51.379367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:51.379615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519036328391156:2067] [node 5] ICS01 socket: 536 reason# EndOfStream
2025-11-11T17:20:51.379670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519036328391138:2067] [node 5] ICP30 unregister session Session# [6:7571519036328391156:2067] VirtualId# [6:7571519036328391148:0]
2025-11-11T17:20:51.379704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519036328391156:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:51.379794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:51.379840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:51.658003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519039047786174:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:51.658050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519039047786175:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:51.688503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519039071944635:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:51.688836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519039071944634:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:51.737906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519039301800964:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:51.737953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519039301800965:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:51.777117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519038021082393:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:51.777165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519038021082394:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:51.777195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519038021082395:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:51.822055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519037927687903:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:51.822100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519037927687904:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:51.827551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519037927687902:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:51.870054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519040541562803:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:51.870103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519040541562804:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:51.870133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519040541562805:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:52.086050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519043342753522:2096] [node 4] ICP01 ready to work
2025-11-11T17:20:52.150523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519039301800964:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(32430504ns) {Configure:104(32421818ns) {ProcessConfigured:135(81612ns) {StartInitialHandshake:157(75782ns) {DropHandshakes:469(2592ns) {DropIncomingHandshake:441(566ns) DropOutgoingHandshake:456(386ns)} GenerateSessionVirtualId:988(1287ns) PrepareNewSessionHandshake:476(1497ns) {SwitchToState:178(545ns)}}}}}
2025-11-11T17:20:52.150991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519043596768283:2066] [node 4] ICP01 ready to work
2025-11-11T17:20:52.128487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519043366911971:2075] [node 4] ICP01 ready to work
2025-11-11T17:20:52.086465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519043342753523:2098] [node 6] ICP01 ready to work
2025-11-11T17:20:52.128834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519043366911972:2077] [node 5] ICP01 ready to work
2025-11-11T17:20:52.183333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519044836530114:2061] [node 5] ICP01 ready to work
2025-11-11T17:20:52.161907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519042316049713:2068] [node 5] ICP01 ready to work
2025-11-11T17:20:52.185229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519042222655217:2063] [node 4] ICP01 ready to work
2025-11-11T17:20:52.129137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519043366911973:2078] [node 6] ICP01 ready to work
2025-11-11T17:20:52.194343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519042316049714:2069] [node 6] ICP01 ready to work
2025-11-11T17:20:52.225510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519043366911971:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(96064136ns) {Configure:104(96059512ns) {ProcessConfigured:135(76087ns) {StartInitialHandshake:157(72703ns) {DropHandshakes:469(5433ns) {DropIncomingHandshake:441(976ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(1465ns) PrepareNewSessionHandshake:476(1839ns) {SwitchToState:178(961ns)}}}}}
2025-11-11T17:20:52.238035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519043342753522:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(151231111ns) {Configure:104(151225742ns) {ProcessConfigured:135(9476831ns) {StartInitialHandshake:157(4027109ns) {DropHandshakes:469(3385ns) {DropIncomingHandshake:441(391ns) DropOutgoingHandshake:456(417ns)} GenerateSessionVirtualId:988(2974ns) PrepareNewSessionHandshake:476(5676ns) {SwitchToState:178(2317ns)}}}}}
2025-11-11T17:20:52.251132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519042222655217:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(56507152ns) {Configure:104(56502631ns) {ProcessConfigured:135(90528ns) {StartInitialHandshake:157(87728ns) {DropHandshakes:469(5305ns) {DropIncomingHandshake:441(598ns) DropOutgoingHandshake:456(324ns)} GenerateSessionVirtualId:988(1725ns) PrepareNewSessionHandshake:476(1691ns) {SwitchToState:178(812ns)}}}}}
2025-11-11T17:20:52.233853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519039071944634:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(118000745ns) {Configure:104(117995040ns) {ProcessConfigured:135(32944965ns) {StartInitialHandshake:157(29323453ns) {DropHandshakes:469(3181ns) {DropIncomingHandshake:441(554ns) DropOutgoingHandshake:456(525ns)} GenerateSessionVirtualId:988(2025ns) PrepareNewSessionHandshake:476(4808ns) {SwitchToState:178(2060ns)}}}}}
2025-11-11T17:20:52.405913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519043342753539:2102] [node 5] ICP01 ready to work
2025-11-11T17:20:52.410348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519040541562803:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(252400396ns) {Configure:104(252395523ns) {ProcessConfigured:135(111885ns) {StartInitialHandshake:157(108369ns) {DropHandshakes:469(2751ns) {DropIncomingHandshake:441(461ns) DropOutgoingHandshake:456(631ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:476(2014ns) {SwitchToState:178(760ns)}}}}}
2025-11-11T17:20:52.503253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519038021082393:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(313568952ns) {Configure:104(313562441ns) {ProcessConfigured:135(67303ns) {StartInitialHandshake:157(64198ns) {DropHandshakes:469(2823ns) {DropIncomingHandshake:441(593ns) DropOutgoingHandshake:456(406ns)} GenerateSessionVirtualId:988(1070ns) PrepareNewSessionHandshake:476(1654ns) {SwitchToState:178(556ns)}}}}}
2025-11-11T17:20:52.499011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519043596768339:2103] [node 5] ICP01 ready to work
2025-11-11T17:20:52.499151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519042222655268:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:52.482167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519043342753539:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25597046ns) {Configure:104(25592699ns) {ProcessConfigured:135(72046ns) {StartInitialHandshake:157(69232ns) {DropHandshakes:469(2724ns) {DropIncomingHandshake:441(622ns) DropOutgoingHandshake:456(499ns)} GenerateSessionVirtualId:988(876ns) PrepareNewSessionHandshake:476(1447ns) {SwitchToState:178(459ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.501667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519043342753523:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(218446380ns) {Configure:104(218435666ns) {ProcessConfigured:135(27939348ns) {StartInitialHandshake:157(27935817ns) {DropHandshakes:469(1304ns) {DropIncomingHandshake:441(389ns) DropOutgoingHandshake:456(212ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:476(6954ns) {SwitchToState:178(2552ns)}}}}}
2025-11-11T17:20:52.531980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519043342753541:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46164
2025-11-11T17:20:52.532507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519043342753548:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46164
2025-11-11T17:20:52.481857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519040541562804:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(220241488ns) {Configure:104(220235869ns) {ProcessConfigured:135(126246161ns) {StartInitialHandshake:157(126242327ns) {DropHandshakes:469(2881ns) {DropIncomingHandshake:441(586ns) DropOutgoingHandshake:456(509ns)} GenerateSessionVirtualId:988(1329ns) PrepareNewSessionHandshake:476(7894ns) {SwitchToState:178(3083ns)}}}}}
2025-11-11T17:20:52.510986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519043366911973:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(276843910ns) {Configure:104(276839288ns) {ProcessConfigured:135(68577ns) {StartInitialHandshake:157(66477ns) {DropHandshakes:469(2239ns) {DropIncomingHandshake:441(436ns) DropOutgoingHandshake:456(350ns)} GenerateSessionVirtualId:988(1408ns) PrepareNewSessionHandshake:476(1258ns) {SwitchToState:178(463ns)}}}}}
2025-11-11T17:20:52.546243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519043366911972:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(320469681ns) {Configure:104(320465594ns) {ProcessConfigured:135(11921972ns) {StartInitialHandshake:157(11918605ns) {DropHandshakes:469(1747ns) {DropIncomingHandshake:441(528ns) DropOutgoingHandshake:456(241ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:476(4855ns) {SwitchToState:178(2912ns)}}}}}
2025-11-11T17:20:52.504065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519042316049713:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(309448299ns) {Configure:104(309443782ns) {ProcessConfigured:135(51442ns) {StartInitialHandshake:157(50009ns) {DropHandshakes:469(1226ns) {DropIncomingHandshake:441(496ns) DropOutgoingHandshake:456(111ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(4043ns) {SwitchToState:178(635ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.553946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519043596768340:2111] [node 6] ICP01 ready to work
2025-11-11T17:20:52.509710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519044836530166:2075] [node 4] ICP01 ready to work
2025-11-11T17:20:52.527743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519038021082395:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24247154ns) {Configure:104(24242172ns) {ProcessConfigured:135(61972ns) {StartInitialHandshake:157(59662ns) {DropHandshakes:469(2978ns) {DropIncomingHandshake:441(583ns) DropOutgoingHandshake:456(412ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:476(1470ns) {SwitchToState:178(471ns)}}}}}
2025-11-11T17:20:52.558969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049764:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
2025-11-11T17:20:52.559256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
2025-11-11T17:20:52.559379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049766:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
2025-11-11T17:20:52.559448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049767:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
2025-11-11T17:20:52.570157Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519042316049767:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.586221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519038021082393:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.568639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519043342753552:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46164
2025-11-11T17:20:52.582045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519039301800965:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(396552153ns) {Configure:104(396547263ns) {ProcessConfigured:135(6187754ns) {StartInitialHandshake:157(6184626ns) {DropHandshakes:469(6086372ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(3820ns)} GenerateSessionVirtualId:988(3169ns) PrepareNewSessionHandshake:476(4367ns) {SwitchToState:178(2789ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.588993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519037927687904:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(431043320ns) {Configure:104(423105931ns) {ProcessConfigured:135(86480ns) {StartInitialHandshake:157(84179ns) {DropHandshakes:469(2370ns) {DropIncomingHandshake:441(502ns) DropOutgoingHandshake:456(423ns)} GenerateSessionVirtualId:988(671ns) PrepareNewSessionHandshake:476(1858ns) {SwitchToState:178(528ns)}}}}}
2025-11-11T17:20:52.598187Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753541:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.593679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519042316049764:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.605349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519043366912010:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-11T17:20:52.605427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519043366912011:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-11T17:20:52.649535Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912011:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.605924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519043342753540:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46164
2025-11-11T17:20:52.637723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753536:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.603925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519043596768346:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-11-11T17:20:52.623999Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519042316049765:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.610652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519037927687903:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20767709ns) {Configure:104(20761434ns) {ProcessConfigured:135(66099ns) {StartInitialHandshake:157(63138ns) {DropHandshakes:469(5133ns) {DropIncomingHandshake:441(554ns) DropOutgoingHandshake:456(388ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:476(1446ns) {SwitchToState:178(414ns)}}}}}
2025-11-11T17:20:52.637779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519043342753536:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.664481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519044836530114:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(219257833ns) {Configure:104(219252437ns) {ProcessConfigured:135(34179148ns) {StartInitialHandshake:157(34176000ns) {DropHandshakes:469(2903ns) {DropIncomingHandshake:441(494ns) DropOutgoingHandshake:456(432ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:476(3915ns) {SwitchToState:178(1524ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.645124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519043596768346:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.694125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519044836530166:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(148743979ns) {Configure:104(148738261ns) {ProcessConfigured:135(23314225ns) {StartInitialHandshake:157(23310478ns) {DropHandshakes:469(5999220ns) {DropIncomingHandshake:441(2819ns) DropOutgoingHandshake:456(475ns)} GenerateSessionVirtualId:988(1867ns) PrepareNewSessionHandshake:476(3643ns) {SwitchToState:178(1790ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.702069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519044836530177:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:52.662488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519038021082395:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.611087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519042222655271:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:52.646515Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753543:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.709832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912010:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.724793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519043366912013:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.728876Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519044836530177:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.729267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519040541562803:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.645511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519039301800964:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.662935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049772:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.762532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519044836530179:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.763144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519044836530180:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:52.763426Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519044836530177:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:52.681665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519042222655268:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(97409918ns) {Configure:104(97402023ns) {ProcessConfigured:135(71471ns) {StartInitialHandshake:157(68813ns) {DropHandshakes:469(2093ns) {DropIncomingHandshake:441(490ns) DropOutgoingHandshake:456(492ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:476(1357ns) {SwitchToState:178(607ns)}}}}}
2025-11-11T17:20:52.773789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519043366912014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-11T17:20:52.646539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519043342753543:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.763727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519040541562803:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:52.645621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519043596768350:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-11-11T17:20:52.697322Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519042316049767:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:52.763814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:52.653796Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753540:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.658283Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519043342753536:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:52.661858Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519043342753543:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:52.682286Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753552:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.682445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519039047786175:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.682520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:52.682570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519039047786175:2049] [node 3] ICP22 created new session: [1:7571519043342753556:2049]
2025-11-11T17:20:52.682620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519043342753556:2049] [node 3] ICS09 handshake done sender: [1:7571519043342753543:2114] self: [1:7571519043342753542:0] peer: [3:7571519043596768351:0] socket: 535 qp: -1
2025-11-11T17:20:52.682659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519043342753556:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:52.682695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:52.683152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519043342753522:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.683186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:52.683225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519043342753522:2096] [node 4] ICP22 created new session: [1:7571519043342753558:2096]
2025-11-11T17:20:52.683279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519043342753558:2096] [node 4] ICS09 handshake done sender: [1:7571519043342753536:2110] self: [1:7571519043342753535:0] peer: [4:7571519042316049771:0] socket: 533 qp: -1
2025-11-11T17:20:52.683321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519043342753558:2096] [node 4] ICS10 traffic start
2025-11-11T17:20:52.683347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:52.690495Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753548:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.690605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519043342753522:2096] [node 4] ICP18 handshake (actor [1:7571519043342753552:2118]) is too old
2025-11-11T17:20:52.728028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519043342753564:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46164
2025-11-11T17:20:52.742579Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753538:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.742609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519043342753538:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.745136Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519043342753538:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:52.745676Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753564:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.649348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519043596768352:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-11-11T17:20:52.798065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519043342753523:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.798168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:52.798222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519043342753523:2098] [node 6] ICP22 created new session: [1:7571519043342753567:2098]
2025-11-11T17:20:52.798288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519043342753567:2098] [node 6] ICS09 handshake done sender: [1:7571519043342753538:2111] self: [1:7571519043342753537:0] peer: [6:7571519044836530178:0] socket: 534 qp: -1
2025-11-11T17:20:52.798329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519043342753567:2098] [node 6] ICS10 traffic start
2025-11-11T17:20:52.798377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:52.805851Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912014:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.813834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519039071944634:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.816195Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366911956:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.816229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519043366911956:2073] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.817342Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519043366911956:2073] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.722522Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:52.763862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519040541562803:2048] [node 1] ICP22 created new session: [6:7571519044836530181:2048]
2025-11-11T17:20:52.724209Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519042222655271:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.737921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519042222655217:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.651927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519043596768350:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.763930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519044836530181:2048] [node 1] ICS09 handshake done sender: [6:7571519044836530177:2115] self: [6:7571519044836530178:0] peer: [1:7571519043342753537:0] socket: 555 qp: -1
2025-11-11T17:20:52.769747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519044836530181:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:52.789808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:52.723640Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519042316049766:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.717731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519043596768345:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-11-11T17:20:52.790307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519040541562803:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(26337936ns) {HandleHandshakeStatus:329(26336370ns) {DropIncomingHandshake:441(1518ns) DropOutgoingHandshake:456(4826ns) SwitchToState:178(661ns) ProcessPendingSessionEvents:495(107312ns) {ForwardSessionEventToSession:612(60115ns) ForwardSessionEventToSession:612(3744ns) ForwardSessionEventToSession:612(5653ns) ForwardSessionEventToSession:612(5634ns) ForwardSessionEventToSession:612(2881ns) ForwardSessionEventToSession:612(6012ns) ForwardSessionEventToSession:612(2513ns) ForwardSessionEventToSession:612(2592ns) ForwardSessionEventToSession:612(4947ns) ForwardSessionEventToSession:612(2765ns)}}}
2025-11-11T17:20:52.827408Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519044836530180:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.829305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519044836530166:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.862165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519044836530185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.849728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519039071944635:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.761924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519042222655278:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:52.723698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519038021082393:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.877812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:52.744215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519042316049757:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.744237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519042316049757:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.751261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:52.751352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519038021082393:2048] [node 1] ICP22 created new session: [4:7571519042316049775:2048]
2025-11-11T17:20:52.751407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519042316049775:2048] [node 1] ICS09 handshake done sender: [4:7571519042316049767:2118] self: [4:7571519042316049771:0] peer: [1:7571519043342753535:0] socket: 538 qp: -1
2025-11-11T17:20:52.751452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519042316049775:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:52.751523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:52.752097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519038021082393:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28120243ns) {HandleHandshakeStatus:329(28117278ns) {DropIncomingHandshake:441(765ns) DropOutgoingHandshake:456(5043ns) SwitchToState:178(914ns) ProcessPendingSessionEvents:495(117237ns) {ForwardSessionEventToSession:612(61820ns) ForwardSessionEventToSession:612(9885ns) ForwardSessionEventToSession:612(3014ns) ForwardSessionEventToSession:612(5889ns) ForwardSessionEventToSession:612(3455ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(6162ns) ForwardSessionEventToSession:612(3082ns) ForwardSessionEventToSession:612(5805ns) ForwardSessionEventToSession:612(5360ns)}}}
2025-11-11T17:20:52.752325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519038021082393:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:52.753377Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519042316049757:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:52.782037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519042316049713:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.782123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:52.782178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519042316049713:2068] [node 5] ICP22 created new session: [4:7571519042316049779:2068]
2025-11-11T17:20:52.782230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519042316049779:2068] [node 5] ICS09 handshake done sender: [4:7571519042316049757:2111] self: [4:7571519042316049756:0] peer: [5:7571519042222655277:0] socket: 540 qp: -1
2025-11-11T17:20:52.782263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519042316049779:2068] [node 5] ICS10 traffic start
2025-11-11T17:20:52.782318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:52.791330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519038021082394:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.822490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049783:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
2025-11-11T17:20:52.827224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049784:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
2025-11-11T17:20:52.837714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [4:7571519042316049713:2068] [node 5] ICP35 event processing took too much time StateWork:293(23888339ns)
2025-11-11T17:20:52.849741Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519042316049763:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.849783Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519042316049763:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.850657Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519042316049784:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.850921Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519042316049764:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:52.851350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519038021082395:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:52.851398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:52.851486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519038021082395:2050] [node 3] ICP22 created new session: [4:7571519042316049787:2050]
2025-11-11T17:20:52.851526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519042316049787:2050] [node 3] ICS09 handshake done sender: [4:7571519042316049764:2115] self: [4:7571519042316049773:0] peer: [3:7571519043596768284:0] socket: 531 qp: -1
2025-11-11T17:20:52.851557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519042316049787:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:52.851613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:52.852035Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519042316049763:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:52.865772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519042316049714:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.873736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:52.879566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519042316049714:2069] [node 6] ICP22 created new session: [4:7571519042316049789:2069]
2025-11-11T17:20:52.893780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519042316049789:2069] [node 6] ICS09 handshake done sender: [4:7571519042316049763:2114] self: [4:7571519042316049762:0] peer: [6:7571519044836530184:0] socket: 565 qp: -1
2025-11-11T17:20:52.893825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519042316049789:2069] [node 6] ICS10 traffic start
2025-11-11T17:20:52.893879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:52.718102Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519043596768285:2070] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.773702Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519042222655278:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.894687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753547:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.894749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519043342753547:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.877880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519039071944635:2049] [node 3] ICP22 created new session: [2:7571519043366912018:2049]
2025-11-11T17:20:52.718133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519043596768285:2070] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.910920Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519044836530180:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.815402Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:52.917689Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519043342753547:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.877937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519043366912018:2049] [node 3] ICS09 handshake done sender: [2:7571519043366911956:2073] self: [2:7571519043366911955:0] peer: [3:7571519043596768356:0] socket: 528 qp: -1
2025-11-11T17:20:52.718754Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:52.946650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519042316049714:2069] [node 6] ICP35 event processing took too much time PendingConnection:278(80557737ns) {HandleHandshakeStatus:329(80555379ns) {DropIncomingHandshake:441(21916ns) DropOutgoingHandshake:456(336ns) SwitchToState:178(401ns) ProcessPendingSessionEvents:495(15178228ns) {ForwardSessionEventToSession:612(15083334ns) ForwardSessionEventToSession:612(12901ns) ForwardSessionEventToSession:612(3801ns) ForwardSessionEventToSession:612(2830ns) ForwardSessionEventToSession:612(6012ns) ForwardSessionEventToSession:612(3629ns) ForwardSessionEventToSession:612(4101ns) ForwardSessionEventToSession:612(6209ns) ForwardSessionEventToSession:612(3736ns) ForwardSessionEventToSession:612(40202ns)}}}
2025-11-11T17:20:52.947112Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:52.970514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519038021082394:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519042316049766:2117] held: no
2025-11-11T17:20:52.970733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519042316049791:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5935
2025-11-11T17:20:52.824924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519042222655217:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519042222655271:2113] held: no
2025-11-11T17:20:52.930636Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:53.002940Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519042316049766:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:52.939103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519039047786174:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.877994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519043366912018:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:52.777997Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519039301800964:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519043596768346:2124] held: no
2025-11-11T17:20:52.937917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519044836530166:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:52.878067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:52.878603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519039071944635:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(35825510ns) {HandleHandshakeStatus:329(28657969ns) {DropIncomingHandshake:441(23383ns) DropOutgoingHandshake:456(396ns) SwitchToState:178(701ns) ProcessPendingSessionEvents:495(115463ns) {ForwardSessionEventToSession:612(59287ns) ForwardSessionEventToSession:612(4006ns) ForwardSessionEventToSession:612(9987ns) ForwardSessionEventToSession:612(2967ns) ForwardSessionEventToSession:612(2545ns) ForwardSessionEventToSession:612(13734ns) ForwardSessionEventToSession:612(3280ns) ForwardSessionEventToSession:612(2685ns) ForwardSessionEventToSession:612(5245ns) ForwardSessionEventToSession:612(2927ns)}}}
2025-11-11T17:20:52.882614Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912000:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.882656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519043366912000:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.891137Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912013:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.891781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519043366912022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-11T17:20:52.892061Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519043366912000:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:52.922249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519043366912024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-11T17:20:52.926257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519043366911971:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.926389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:52.926443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519043366911971:2075] [node 4] ICP22 created new session: [2:7571519043366912025:2075]
2025-11-11T17:20:52.926497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519043366912025:2075] [node 4] ICS09 handshake done sender: [2:7571519043366912000:2111] self: [2:7571519043366911999:0] peer: [4:7571519042316049781:0] socket: 530 qp: -1
2025-11-11T17:20:52.926540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519043366912025:2075] [node 4] ICS10 traffic start
2025-11-11T17:20:52.926576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:52.930279Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519043366912014:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:52.958543Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:52.959593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519039071944634:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:52.959683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:52.959745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519039071944634:2048] [node 1] ICP22 created new session: [2:7571519043366912028:2048]
2025-11-11T17:20:52.959825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519043366912028:2048] [node 1] ICS09 handshake done sender: [2:7571519043366912014:2119] self: [2:7571519043366912016:0] peer: [1:7571519043342753546:0] socket: 562 qp: -1
2025-11-11T17:20:52.959864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519043366912028:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:52.959928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:52.960201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519039071944634:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:53.012378Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912004:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.012426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519043366912004:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.862622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519042222655280:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:53.015679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519038021082394:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:52.939180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:53.022712Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519043366912004:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:53.023485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519043366911973:2078] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.882146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519042222655282:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:52.893805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519042222655281:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:52.918276Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519042222655282:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.918394Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519042222655271:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:52.977674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519042222655280:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.996074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519042222655217:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:52.996193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:52.996254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519042222655217:2063] [node 4] ICP22 created new session: [5:7571519042222655284:2063]
2025-11-11T17:20:52.996310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519042222655284:2063] [node 4] ICS09 handshake done sender: [5:7571519042222655271:2113] self: [5:7571519042222655277:0] peer: [4:7571519042316049756:0] socket: 546 qp: -1
2025-11-11T17:20:52.996347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519042222655284:2063] [node 4] ICS10 traffic start
2025-11-11T17:20:52.996426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:53.029798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:53.029890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519043366911973:2078] [node 6] ICP22 created new session: [2:7571519047661879327:2078]
2025-11-11T17:20:53.029969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519047661879327:2078] [node 6] ICS09 handshake done sender: [2:7571519043366912004:2114] self: [2:7571519043366912003:0] peer: [6:7571519049131497486:0] socket: 576 qp: -1
2025-11-11T17:20:53.030009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519047661879327:2078] [node 6] ICS10 traffic start
2025-11-11T17:20:53.030083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:53.030783Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912022:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.938018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:52.965957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519039047786174:2048] [node 2] ICP22 created new session: [1:7571519043342753570:2048]
2025-11-11T17:20:52.781686Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519043596768346:2124] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:52.938072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519044836530166:2075] [node 4] ICP22 created new session: [6:7571519044836530187:2075]
2025-11-11T17:20:53.036432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519037927687904:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:53.037219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519037927687902:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.058046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519046517622584:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:52.814526Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519043596768345:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:52.814945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519039301800965:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:52.837745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519043596768285:2070] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:52.866160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519043596768283:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:52.866228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:52.866278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519043596768283:2066] [node 4] ICP22 created new session: [3:7571519043596768359:2066]
2025-11-11T17:20:52.866324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519043596768359:2066] [node 4] ICS09 handshake done sender: [3:7571519043596768285:2070] self: [3:7571519043596768284:0] peer: [4:7571519042316049773:0] socket: 524 qp: -1
2025-11-11T17:20:52.866357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519043596768359:2066] [node 4] ICS10 traffic start
2025-11-11T17:20:52.866396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:52.868854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519043596768357:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-11-11T17:20:52.886098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519039301800964:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(49844969ns) {HandleHandshakeStatus:395(49840783ns) {LogHandshakeFail:472(8240496ns) {UpdateErrorStateLog:544(6704ns)} DropOutgoingHandshake:456(3833ns) ProcessPendingSessionEvents:495(17105957ns) {DropSessionEvent:512(14980ns) DropSessionEvent:512(4663ns) DropSessionEvent:512(5136ns) DropSessionEvent:512(25692ns) DropSessionEvent:512(5935ns) DropSessionEvent:512(4267ns) DropSessionEvent:512(6049ns) DropSessionEvent:512(6540127ns) DropSessionEvent:512(6236ns) DropSessionEvent:512(4494ns)}}}
2025-11-11T17:20:52.914269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519039301800964:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:52.914357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:52.921824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519039301800964:2048] [node 1] ICP22 created new session: [3:7571519043596768362:2048]
2025-11-11T17:20:52.933774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519043596768358:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-11-11T17:20:52.945893Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:52.950709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519043596768362:2048] [node 1] ICS09 handshake done sender: [3:7571519043596768346:2124] self: [3:7571519043596768351:0] peer: [1:7571519043342753542:0] socket: 545 qp: -1
2025-11-11T17:20:52.950803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519043596768362:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:52.965863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:52.997022Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:53.019911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519039301800964:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(91455550ns) {HandleHandshakeStatus:329(91452678ns) {DropIncomingHandshake:441(1204ns) DropOutgoingHandshake:456(535ns) SwitchToState:178(464ns) ProcessPendingSessionEvents:495(1325ns)}}
2025-11-11T17:20:53.023210Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519043596768345:2123] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:53.023976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519039301800965:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519043596768345:2123] held: no
2025-11-11T17:20:53.024073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519039301800965:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:53.024139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:53.024182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519039301800965:2049] [node 2] ICP22 created new session: [3:7571519047891735661:2049]
2025-11-11T17:20:53.024234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519047891735661:2049] [node 2] ICS09 handshake done sender: [3:7571519043596768345:2123] self: [3:7571519043596768356:0] peer: [2:7571519043366911955:0] socket: 544 qp: -1
2025-11-11T17:20:53.024268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519047891735661:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:53.024294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:53.025291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519047891735660:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-11-11T17:20:53.027170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519043596768283:2066] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:53.033242Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519047891735660:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.059686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519043596768344:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.059744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519043596768344:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:52.998552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519043342753570:2048] [node 2] ICS09 handshake done sender: [1:7571519043342753547:2116] self: [1:7571519043342753546:0] peer: [2:7571519043366912016:0] socket: 561 qp: -1
2025-11-11T17:20:53.047173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:53.063120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519038021082394:2049] [node 2] ICP22 created new session: [4:7571519046611017088:2049]
2025-11-11T17:20:52.938121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519044836530187:2075] [node 4] ICS09 handshake done sender: [6:7571519044836530180:2117] self: [6:7571519044836530184:0] peer: [4:7571519042316049762:0] socket: 566 qp: -1
2025-11-11T17:20:52.938161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519044836530187:2075] [node 4] ICS10 traffic start
2025-11-11T17:20:52.938204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:52.938455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519044836530166:2075] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:52.938813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519044836530186:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:52.973686Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519044836530186:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.002296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519040541562804:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:53.046014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519049131497487:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:53.012489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519043342753570:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:53.079134Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519043596768344:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:53.073042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519046611017088:2049] [node 2] ICS09 handshake done sender: [4:7571519042316049766:2117] self: [4:7571519042316049781:0] peer: [2:7571519043366911999:0] socket: 537 qp: -1
2025-11-11T17:20:53.070826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519046517622584:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.015369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:53.032158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519039047786174:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(92805878ns) {HandleHandshakeStatus:329(92802838ns) {DropIncomingHandshake:441(32050ns) DropOutgoingHandshake:456(408ns) SwitchToState:178(273ns) ProcessPendingSessionEvents:495(6137180ns) {ForwardSessionEventToSession:612(6066152ns) ForwardSessionEventToSession:612(9226ns) ForwardSessionEventToSession:612(3866ns) ForwardSessionEventToSession:612(3279ns) ForwardSessionEventToSession:612(13599ns) ForwardSessionEventToSession:612(3122ns) ForwardSessionEventToSession:612(4183ns) ForwardSessionEventToSession:612(11727ns) ForwardSessionEventToSession:612(5633ns) ForwardSessionEventToSession:612(5888ns)}}}
2025-11-11T17:20:53.037494Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519043342753545:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.037521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519043342753545:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:53.074539Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519043342753545:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:53.083184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519037927687903:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:53.073124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519046611017088:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:53.073188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:53.077964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519038021082394:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(62188885ns) {HandleHandshakeStatus:329(62184629ns) {DropIncomingHandshake:441(767ns) DropOutgoingHandshake:456(426ns) SwitchToState:178(1230ns) ProcessPendingSessionEvents:495(934ns)}}
2025-11-11T17:20:53.079609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519043596768339:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:53.079710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:53.079769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519043596768339:2103] [node 5] ICP22 created new session: [3:7571519047891735665:2103]
2025-11-11T17:20:53.079821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519047891735665:2103] [node 5] ICS09 handshake done sender: [3:7571519043596768344:2122] self: [3:7571519043596768343:0] peer: [5:7571519046517622582:0] socket: 542 qp: -1
2025-11-11T17:20:53.079858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519047891735665:2103] [node 5] ICS10 traffic start
2025-11-11T17:20:53.079890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:53.097874Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519043366912009:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.094247Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:53.106609Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:53.097925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519043366912009:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:53.109211Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519043366912009:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:53.109712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519043366911972:2077] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:53.117938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:53.118009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519043366911972:2077] [node 5] ICP22 created new session: [2:7571519047661879331:2077]
2025-11-11T17:20:53.118068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519047661879331:2077] [node 5] ICS09 handshake done sender: [2:7571519043366912009:2115] self: [2:7571519043366912008:0] peer: [5:7571519046517622585:0] socket: 543 qp: -1
2025-11-11T17:20:53.118113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519047661879331:2077] [node 5] ICS10 traffic start
2025-11-11T17:20:53.118155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:53.107638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519043342753539:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:53.107727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:53.107786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519043342753539:2102] [node 5] ICP22 created new session: [1:7571519047637720869:2102]
2025-11-11T17:20:53.107838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519047637720869:2102] [node 5] ICS09 handshake done sender: [1:7571519043342753545:2115] self: [1:7571519043342753544:0] peer: [5:7571519046517622583:0] socket: 560 qp: -1
2025-11-11T17:20:53.107889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519047637720869:2102] [node 5] ICS10 traffic start
2025-11-11T17:20:53.107980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:53.142003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519037927687904:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519042222655282:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.108866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519040541562804:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519044836530186:2119] held: no
2025-11-11T17:20:53.143709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519037927687904:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(16843146ns) {HandleHandshakeStatus:395(5357308ns) {LogHandshakeFail:472(17180ns) {UpdateErrorStateLog:544(6200ns)} DropOutgoingHandshake:456(2319ns) ProcessPendingSessionEvents:495(1407667ns) {DropSessionEvent:512(1341655ns) DropSessionEvent:512(8926ns) DropSessionEvent:512(7039ns) DropSessionEvent:512(5568ns) DropSessionEvent:512(5241ns) DropSessionEvent:512(7612ns) DropSessionEvent:512(6180ns) DropSessionEvent:512(4740ns) DropSessionEvent:512(5453ns) DropSessionEvent:512(8279ns)}}}
2025-11-11T17:20:53.143856Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:53.126466Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519044836530186:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:53.142219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519040541562804:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:53.142292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:53.142343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519040541562804:2049] [node 2] ICP22 created new session: [6:7571519049131497489:2049]
2025-11-11T17:20:53.142412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519049131497489:2049] [node 2] ICS09 handshake done sender: [6:7571519044836530186:2119] self: [6:7571519049131497486:0] peer: [2:7571519043366912003:0] socket: 578 qp: -1
2025-11-11T17:20:53.142445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519049131497489:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:53.142505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:53.143003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519049131497488:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:53.145305Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.146860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519046517622586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.147326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519046517622587:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:53.147536Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519049131497488:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.147666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519040541562805:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:53.148335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519037927687903:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519046517622584:2120] held: no
2025-11-11T17:20:53.148513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519037927687902:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519042222655280:2117] held: no
2025-11-11T17:20:53.148747Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519042222655280:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.149277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519046517622588:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-11T17:20:53.149916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519042222655282:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.150402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519049131497492:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:53.150701Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519046517622584:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:53.150961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519037927687902:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:53.151042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:53.151101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519037927687902:2048] [node 1] ICP22 created new session: [5:7571519046517622590:2048]
2025-11-11T17:20:53.151142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519046517622590:2048] [node 1] ICS09 handshake done sender: [5:7571519042222655280:2117] self: [5:7571519046517622583:0] peer: [1:7571519043342753544:0] socket: 563 qp: -1
2025-11-11T17:20:53.151176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519046517622590:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:53.151247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:53.151626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519037927687903:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:53.151714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:53.151750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519037927687903:2049] [node 2] ICP22 created new session: [5:7571519046517622592:2049]
2025-11-11T17:20:53.151797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519046517622592:2049] [node 2] ICS09 handshake done sender: [5:7571519046517622584:2120] self: [5:7571519046517622585:0] peer: [2:7571519043366912008:0] socket: 575 qp: -1
2025-11-11T17:20:53.151824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519046517622592:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:53.151872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:53.153625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519037927687904:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:53.153720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:53.153761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519037927687904:2050] [node 3] ICP22 created new session: [5:7571519046517622594:2050]
2025-11-11T17:20:53.153799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519046517622594:2050] [node 3] ICS09 handshake done sender: [5:7571519042222655282:2119] self: [5:7571519046517622582:0] peer: [3:7571519043596768343:0] socket: 570 qp: -1
2025-11-11T17:20:53.153825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519046517622594:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:53.153853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:53.153708Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519043596768348:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.153752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519043596768348:2125] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.154843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519049131497493:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:53.155029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519043596768348:2125] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:53.155362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519043596768340:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:53.155454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:53.155515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519043596768340:2111] [node 6] ICP22 created new session: [3:7571519047891735668:2111]
2025-11-11T17:20:53.155560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519047891735668:2111] [node 6] ICS09 handshake done sender: [3:7571519043596768348:2125] self: [3:7571519043596768347:0] peer: [6:7571519049131497491:0] socket: 571 qp: -1
2025-11-11T17:20:53.155592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519047891735668:2111] [node 6] ICS10 traffic start
2025-11-11T17:20:53.155624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:53.169725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519049131497488:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:53.184062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519040541562805:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:53.184161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:53.184220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519040541562805:2050] [node 3] ICP22 created new session: [6:7571519049131497494:2050]
2025-11-11T17:20:53.184272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519049131497494:2050] [node 3] ICS09 handshake done sender: [6:7571519049131497488:2121] self: [6:7571519049131497491:0] peer: [3:7571519043596768347:0] socket: 581 qp: -1
2025-11-11T17:20:53.184311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519049131497494:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:53.184368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:53.185042Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:53.185790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519040541562805:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:53.186592Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519049131497492:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.186726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519044836530114:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:53.187238Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519042222655276:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:53.187279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519042222655276:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:53.188357Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519042222655276:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:53.188690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519042222655268:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:53.188799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:53.188856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519042222655268:2067] [node 6] ICP22 created new session: [5:7571519046517622597:2067]
2025-11-11T17:20:53.188906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519046517622597:2067] [node 6] ICS09 handshake done sender: [5:7571519042222655276:2115] self: [5:7571519042222655275:0] peer: [6:7571519049131497496:0] socket: 522 qp: -1
2025-11-11T17:20:53.188945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519046517622597:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:53.188980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:53.192460Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:53.193409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519044836530114:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519049131497492:2122] held: no
2025-11-11T17:20:53.193644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519049131497497:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21883
2025-11-11T17:20:53.194210Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519049131497492:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:53.194411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519044836530114:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:53.194483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:53.194534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519044836530114:2061] [node 5] ICP22 created new session: [6:7571519049131497498:2061]
2025-11-11T17:20:53.194583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519049131497498:2061] [node 5] ICS09 handshake done sender: [6:7571519049131497492:2122] self: [6:7571519049131497496:0] peer: [5:7571519042222655275:0] socket: 526 qp: -1
2025-11-11T17:20:53.194623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519049131497498:2061] [node 5] ICS10 traffic start
2025-11-11T17:20:53.194659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:53.218150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:53.218364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519043596768362:2048] [node 1] ICS01 socket: 545 reason# EndOfStream
2025-11-11T17:20:53.218397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519039301800964:2048] [node 1] ICP30 unregister session Session# [3:7571519043596768362:2048] VirtualId# [3:7571519043596768351:0]
2025-11-11T17:20:53.218424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519043596768362:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.218477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:53.219172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:53.217754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:53.217935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519043366912028:2048] [node 1] ICS01 socket: 562 reason# EndOfStream
2025-11-11T17:20:53.217969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519039071944634:2048] [node 1] ICP30 unregister session Session# [2:7571519043366912028:2048] VirtualId# [2:7571519043366912016:0]
2025-11-11T17:20:53.217993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519043366912028:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.218070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:53.218117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:53.223740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:53.223938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519044836530181:2048] [node 1] ICS07 socket disconnect 555 reason# EndOfStream
2025-11-11T17:20:53.223963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519044836530181:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.224036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:53.224105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519044836530181:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:53.224859Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Couldn't connect to any resolved address
2025-11-11T17:20:53.224937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519040541562803:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:53.225290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519044836530181:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:53.225327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519040541562803:2048] [node 1] ICP30 unregister session Session# [6:7571519044836530181:2048] VirtualId# [6:7571519044836530178:0]
2025-11-11T17:20:53.225367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:53.225418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519040541562803:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:53.225451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21684) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:53.225699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:53.223640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:53.223879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519042316049775:2048] [node 1] ICS01 socket: 538 reason# EndOfStream
2025-11-11T17:20:53.227093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519046517622590:2048] [node 1] ICS01 socket: 563 reason# EndOfStream
2025-11-11T17:20:53.227140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519037927687902:2048] [node 1] ICP30 unregister session Session# [5:7571519046517622590:2048] VirtualId# [5:7571519046517622583:0]
2025-11-11T17:20:53.227173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519046517622590:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.227243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:53.227277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:53.223920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519038021082393:2048] [node 1] ICP30 unregister session Session# [4:7571519042316049775:2048] VirtualId# [4:7571519042316049771:0]
2025-11-11T17:20:53.223953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519042316049775:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.224028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:53.224067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:53.236586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519040541562803:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:53.301499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:53.301803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519047891735661:2049] [node 2] ICS01 socket: 544 reason# EndOfStream
2025-11-11T17:20:53.301851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519039301800965:2049] [node 2] ICP30 unregister session Session# [3:7571519047891735661:2049] VirtualId# [3:7571519043596768356:0]
2025-11-11T17:20:53.301892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519047891735661:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.302017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:53.302065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:53.305886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:53.306130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519046611017088:2049] [node 2] ICS01 socket: 537 reason# EndOfStream
2025-11-11T17:20:53.306164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519038021082394:2049] [node 2] ICP30 unregister session Session# [4:7571519046611017088:2049] VirtualId# [4:7571519042316049781:0]
2025-11-11T17:20:53.306200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519046611017088:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.306297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:53.306334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:53.305760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:53.305999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519046517622592:2049] [node 2] ICS01 socket: 575 reason# EndOfStream
2025-11-11T17:20:53.306037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519037927687903:2049] [node 2] ICP30 unregister session Session# [5:7571519046517622592:2049] VirtualId# [5:7571519046517622585:0]
2025-11-11T17:20:53.306075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519046517622592:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.306140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:53.306185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:53.310039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:53.310278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519049131497489:2049] [node 2] ICS01 socket: 578 reason# EndOfStream
2025-11-11T17:20:53.310327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519040541562804:2049] [node 2] ICP30 unregister session Session# [6:7571519049131497489:2049] VirtualId# [6:7571519049131497486:0]
2025-11-11T17:20:53.310364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519049131497489:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.310462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:53.310502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:53.369048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:53.369345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519042316049787:2050] [node 3] ICS01 socket: 531 reason# EndOfStream
2025-11-11T17:20:53.369388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519038021082395:2050] [node 3] ICP30 unregister session Session# [4:7571519042316049787:2050] VirtualId# [4:7571519042316049773:0]
2025-11-11T17:20:53.369421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519042316049787:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.369514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:53.369561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:53.372586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:53.372758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519046517622594:2050] [node 3] ICS01 socket: 570 reason# EndOfStream
2025-11-11T17:20:53.372797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519037927687904:2050] [node 3] ICP30 unregister session Session# [5:7571519046517622594:2050] VirtualId# [5:7571519046517622582:0]
2025-11-11T17:20:53.372825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519046517622594:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.372915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:53.372963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:53.374519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:53.374739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519049131497494:2050] [node 3] ICS01 socket: 581 reason# EndOfStream
2025-11-11T17:20:53.374776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519040541562805:2050] [node 3] ICP30 unregister session Session# [6:7571519049131497494:2050] VirtualId# [6:7571519049131497491:0]
2025-11-11T17:20:53.374805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519049131497494:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.374876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:53.375026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:53.469286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:53.469540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519042222655284:2063] [node 4] ICS01 socket: 546 reason# EndOfStream
2025-11-11T17:20:53.469581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519042222655217:2063] [node 4] ICP30 unregister session Session# [5:7571519042222655284:2063] VirtualId# [5:7571519042222655277:0]
2025-11-11T17:20:53.469628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519042222655284:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.469703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:53.469749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:53.473471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:53.475999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519044836530187:2075] [node 4] ICS01 socket: 566 reason# EndOfStream
2025-11-11T17:20:53.476091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519044836530166:2075] [node 4] ICP30 unregister session Session# [6:7571519044836530187:2075] VirtualId# [6:7571519044836530184:0]
2025-11-11T17:20:53.476145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519044836530187:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.476313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:53.476402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:53.593279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:53.593674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519049131497498:2061] [node 5] ICS01 socket: 526 reason# EndOfStream
2025-11-11T17:20:53.593754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519044836530114:2061] [node 5] ICP30 unregister session Session# [6:7571519049131497498:2061] VirtualId# [6:7571519049131497496:0]
2025-11-11T17:20:53.593843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519049131497498:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:53.594044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:53.594128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:53.749897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519048862617674:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:53.750494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519048862617673:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:53.778387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519046573640052:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:53.778440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519046573640053:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:53.808251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519047677090060:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:53.808296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519047677090061:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:53.832213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519049859473647:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:53.832259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519049859473648:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:53.832287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519049859473649:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:53.863601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519050212177141:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:53.863725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519050212177142:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:53.863791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519050212177143:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:53.896383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519048917570326:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:53.896429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519048917570327:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:54.272419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519050868607408:2107] [node 5] ICP01 ready to work
2025-11-11T17:20:54.318489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519046573640052:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(45770731ns) {Configure:104(45765479ns) {ProcessConfigured:135(99686ns) {StartInitialHandshake:157(97588ns) {DropHandshakes:469(3089ns) {DropIncomingHandshake:441(718ns) DropOutgoingHandshake:456(685ns)} GenerateSessionVirtualId:988(1780ns) PrepareNewSessionHandshake:476(1794ns) {SwitchToState:178(694ns)}}}}}
2025-11-11T17:20:54.373700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519051972057396:2078] [node 5] ICP01 ready to work
2025-11-11T17:20:53.896458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519048917570328:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:54.473987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519053157585041:2107] [node 4] ICP01 ready to work
2025-11-11T17:20:54.474397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519053157585042:2108] [node 6] ICP01 ready to work
2025-11-11T17:20:54.474737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519053157585043:2109] [node 5] ICP01 ready to work
2025-11-11T17:20:54.473675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519054154440989:2065] [node 6] ICP01 ready to work
2025-11-11T17:20:54.449959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519050212177141:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(175006910ns) {Configure:104(175001978ns) {ProcessConfigured:135(39906864ns) {StartInitialHandshake:157(39903950ns) {DropHandshakes:469(2256ns) {DropIncomingHandshake:441(363ns) DropOutgoingHandshake:456(405ns)} GenerateSessionVirtualId:988(740ns) PrepareNewSessionHandshake:476(8109ns) {SwitchToState:178(2957ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.256437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519053212537641:2063] [node 4] ICP01 ready to work
2025-11-11T17:20:54.409713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519051972057409:2077] [node 4] ICP01 ready to work
2025-11-11T17:20:54.318871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519050868607412:2108] [node 4] ICP01 ready to work
2025-11-11T17:20:54.484730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519050868607422:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52264
2025-11-11T17:20:54.257506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519053212537644:2067] [node 5] ICP01 ready to work
2025-11-11T17:20:54.478035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519051972057422:2091] [node 6] ICP01 ready to work
2025-11-11T17:20:54.566668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519048862617673:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(308645773ns) {Configure:104(308639764ns) {ProcessConfigured:135(76002ns) {StartInitialHandshake:157(72729ns) {DropHandshakes:469(3919ns) {DropIncomingHandshake:441(1131ns) DropOutgoingHandshake:456(601ns)} GenerateSessionVirtualId:988(1235ns) PrepareNewSessionHandshake:476(1524ns) {SwitchToState:178(235ns)}}}}}
2025-11-11T17:20:54.565807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519054507144509:2082] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.586427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519053157585043:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17598643ns) {Configure:104(17594282ns) {ProcessConfigured:135(83464ns) {StartInitialHandshake:157(80149ns) {DropHandshakes:469(4178ns) {DropIncomingHandshake:441(907ns) DropOutgoingHandshake:456(1069ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:476(5421ns) {SwitchToState:178(942ns)}}}}}
2025-11-11T17:20:54.586908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519053157585044:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55310
2025-11-11T17:20:54.587100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519053157585045:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55310
2025-11-11T17:20:54.562556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519050868607423:2105] [node 6] ICP01 ready to work
2025-11-11T17:20:54.602486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519054507144510:2083] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.581813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519054154441007:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.570301Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607422:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.638184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519053212537641:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(381387041ns) {Configure:104(381380914ns) {ProcessConfigured:135(88641ns) {StartInitialHandshake:157(84084ns) {DropHandshakes:469(3912ns) {DropIncomingHandshake:441(725ns) DropOutgoingHandshake:456(794ns)} GenerateSessionVirtualId:988(1268ns) PrepareNewSessionHandshake:476(2254ns) {SwitchToState:178(728ns)}}}}}
2025-11-11T17:20:54.594386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519053157585041:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(119061345ns) {Configure:104(119054065ns) {ProcessConfigured:135(133594ns) {StartInitialHandshake:157(129951ns) {DropHandshakes:469(3924ns) {DropIncomingHandshake:441(940ns) DropOutgoingHandshake:456(523ns)} GenerateSessionVirtualId:988(980ns) PrepareNewSessionHandshake:476(2236ns) {SwitchToState:178(786ns)}}}}}
2025-11-11T17:20:54.619636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585045:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.619727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585044:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.607321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519049859473648:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(130563330ns) {Configure:104(130555235ns) {ProcessConfigured:135(83143ns) {StartInitialHandshake:157(79141ns) {DropHandshakes:469(3717ns) {DropIncomingHandshake:441(870ns) DropOutgoingHandshake:456(625ns)} GenerateSessionVirtualId:988(1283ns) PrepareNewSessionHandshake:476(2262ns) {SwitchToState:178(979ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.622211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519050868607424:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52264
2025-11-11T17:20:54.710107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519053212537644:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(452295763ns) {Configure:104(452290544ns) {ProcessConfigured:135(23983458ns) {StartInitialHandshake:157(23979309ns) {DropHandshakes:469(3744ns) {DropIncomingHandshake:441(781ns) DropOutgoingHandshake:456(642ns)} GenerateSessionVirtualId:988(1430ns) PrepareNewSessionHandshake:476(5195ns) {SwitchToState:178(2823ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.656410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519054154441006:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2609
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.722115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519053157585059:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55310
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.738068Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585059:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.738418Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585047:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.738453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519053157585047:2121] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:54.681962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519054507144511:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:54.656889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519054154441015:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2609
2025-11-11T17:20:54.730811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519053212537698:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:54.745827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519054507144511:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.766120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519051972057409:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(255876167ns) {Configure:104(255867158ns) {ProcessConfigured:135(31431843ns) {StartInitialHandshake:157(31427792ns) {DropHandshakes:469(8101074ns) {DropIncomingHandshake:441(2944ns) DropOutgoingHandshake:456(570ns)} GenerateSessionVirtualId:988(3416ns) PrepareNewSessionHandshake:476(6800ns) {SwitchToState:178(3010ns)}}}}}
2025-11-11T17:20:54.778357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519047677090061:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(515836792ns) {Configure:104(515831246ns) {ProcessConfigured:135(52123135ns) {StartInitialHandshake:157(52118779ns) {DropHandshakes:469(3209ns) {DropIncomingHandshake:441(732ns) DropOutgoingHandshake:456(702ns)} GenerateSessionVirtualId:988(1268ns) PrepareNewSessionHandshake:476(4598ns) {SwitchToState:178(1986ns)}}}}}
2025-11-11T17:20:54.731412Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607424:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.754026Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519053157585047:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.805644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585049:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.805723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519053157585049:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:54.782254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519050212177141:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:54.786754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519047677090060:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(308778187ns) {Configure:104(308766146ns) {ProcessConfigured:135(32539071ns) {StartInitialHandshake:157(32534552ns) {DropHandshakes:469(4063ns) {DropIncomingHandshake:441(870ns) DropOutgoingHandshake:456(666ns)} GenerateSessionVirtualId:988(1338ns) PrepareNewSessionHandshake:476(5094ns) {SwitchToState:178(2189ns)}}}}}
2025-11-11T17:20:54.687052Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519054154441006:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.731637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607414:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.738134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519053212537699:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:54.741094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519053212537698:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.741216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519048917570326:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:54.731673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519050868607414:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:54.791290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519054507144510:2083] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(129027201ns) {Configure:104(129022278ns) {ProcessConfigured:135(68074ns) {StartInitialHandshake:157(64461ns) {DropHandshakes:469(3867ns) {DropIncomingHandshake:441(1159ns) DropOutgoingHandshake:456(689ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:476(2149ns) {SwitchToState:178(847ns)}}}}}
2025-11-11T17:20:54.702955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519054154441007:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(93757124ns) {Configure:104(93752481ns) {ProcessConfigured:135(92320ns) {StartInitialHandshake:157(88717ns) {DropHandshakes:469(3318ns) {DropIncomingHandshake:441(942ns) DropOutgoingHandshake:456(625ns)} GenerateSessionVirtualId:988(1152ns) PrepareNewSessionHandshake:476(2379ns) {SwitchToState:178(611ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.797898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519050212177141:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28143839ns) {IncomingHandshake:261(3466108ns) {DropIncomingHandshake:441(1488ns) PrepareNewSessionHandshake:476(3119ns) {SwitchToState:178(2112ns)} GenerateSessionVirtualId:988(816ns) IssueIncomingHandshakeReply:193(52640ns) {SwitchToState:178(101ns)}}}
2025-11-11T17:20:54.843125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519054507144509:2082] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(180885248ns) {Configure:104(180877214ns) {ProcessConfigured:135(77625ns) {StartInitialHandshake:157(74600ns) {DropHandshakes:469(2299ns) {DropIncomingHandshake:441(481ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(1018ns) PrepareNewSessionHandshake:476(2070ns) {SwitchToState:178(577ns)}}}}}
2025-11-11T17:20:54.867505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519054507144517:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:54.837975Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585051:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.703069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519049859473648:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:54.733236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519050868607426:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52264
2025-11-11T17:20:54.733466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519046573640052:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.877776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519048917570327:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(476881289ns) {Configure:104(476876101ns) {ProcessConfigured:135(24101774ns) {StartInitialHandshake:157(15979753ns) {DropHandshakes:469(5074ns) {DropIncomingHandshake:441(896ns) DropOutgoingHandshake:456(510ns)} GenerateSessionVirtualId:988(2082ns) PrepareNewSessionHandshake:476(6160ns) {SwitchToState:178(3103ns)}}}}}
2025-11-11T17:20:54.740239Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519054154441015:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.878802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519051972057429:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33079
2025-11-11T17:20:54.838026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519053157585051:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:54.880016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519053212537699:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.879019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519051972057431:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33079
2025-11-11T17:20:54.838936Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519053157585049:2122] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:54.754433Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519050868607414:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.740881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519054154441024:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2609
2025-11-11T17:20:54.914261Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:54.934015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519053212537703:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:54.787579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519049859473647:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(435585505ns) {Configure:104(435580047ns) {ProcessConfigured:135(49405040ns) {StartInitialHandshake:157(49401893ns) {DropHandshakes:469(4070ns) {DropIncomingHandshake:441(1148ns) DropOutgoingHandshake:456(598ns)} GenerateSessionVirtualId:988(1516ns) PrepareNewSessionHandshake:476(4259ns) {SwitchToState:178(1763ns)}}}}}
2025-11-11T17:20:54.841647Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519054154441006:2109] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:54.852682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519049859473648:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:54.852801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:54.852871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519049859473648:2049] [node 2] ICP22 created new session: [4:7571519054154441026:2049]
2025-11-11T17:20:54.852916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519054154441026:2049] [node 2] ICS09 handshake done sender: [4:7571519054154441006:2109] self: [4:7571519054154441022:0] peer: [2:7571519050868607413:0] socket: 532 qp: -1
2025-11-11T17:20:54.852956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519054154441026:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:54.852995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:54.894228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519054154441029:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2609
2025-11-11T17:20:54.900487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519054154441028:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2609
2025-11-11T17:20:54.917746Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519054154441029:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.922189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519049859473647:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:54.806430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519050868607423:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(187446302ns) {Configure:104(187438083ns) {ProcessConfigured:135(86019ns) {StartInitialHandshake:157(82680ns) {DropHandshakes:469(922ns) {DropIncomingHandshake:441(451ns) DropOutgoingHandshake:456(220ns)} GenerateSessionVirtualId:988(374ns) PrepareNewSessionHandshake:476(2380ns) {SwitchToState:178(860ns)}}}}}
2025-11-11T17:20:54.879137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519051972057432:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33079
2025-11-11T17:20:54.957721Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:54.966821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519050212177141:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519054507144511:2113] held: no
2025-11-11T17:20:54.967162Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519054507144511:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.839388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519048862617673:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:54.985690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519054507144518:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:54.806755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607426:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.807335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519050868607432:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52264
2025-11-11T17:20:54.834008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519050868607408:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(508294862ns) {Configure:104(508274071ns) {ProcessConfigured:135(52373114ns) {StartInitialHandshake:157(52369388ns) {DropHandshakes:469(3703ns) {DropIncomingHandshake:441(1242ns) DropOutgoingHandshake:456(670ns)} GenerateSessionVirtualId:988(1611ns) PrepareNewSessionHandshake:476(2466ns) {SwitchToState:178(111ns)}}}}}
2025-11-11T17:20:54.866108Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:54.867156Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519050868607424:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:54.867798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519046573640052:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519050868607424:2114] held: no
2025-11-11T17:20:54.867898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519046573640052:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:54.873734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:54.890169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519050868607412:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(33915533ns) {IncomingHandshake:261(33913172ns) {DropIncomingHandshake:441(572ns) PrepareNewSessionHandshake:476(2552ns) {SwitchToState:178(957ns)} GenerateSessionVirtualId:988(932ns) IssueIncomingHandshakeReply:193(6904ns)}}
2025-11-11T17:20:54.890248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519046573640052:2048] [node 1] ICP22 created new session: [2:7571519050868607435:2048]
2025-11-11T17:20:54.913809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519050868607435:2048] [node 1] ICS09 handshake done sender: [2:7571519050868607424:2114] self: [2:7571519050868607428:0] peer: [1:7571519053157585046:0] socket: 542 qp: -1
2025-11-11T17:20:54.915593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519050868607412:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:54.915661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:54.915715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519050868607412:2108] [node 4] ICP22 created new session: [2:7571519050868607437:2108]
2025-11-11T17:20:54.915800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519050868607437:2108] [node 4] ICS09 handshake done sender: [2:7571519050868607414:2110] self: [2:7571519050868607413:0] peer: [4:7571519054154441022:0] socket: 529 qp: -1
2025-11-11T17:20:54.915843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519050868607437:2108] [node 4] ICS10 traffic start
2025-11-11T17:20:54.915886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:54.917733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519050868607435:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:54.917792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:54.926052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519046573640052:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(58027359ns) {HandleHandshakeStatus:329(58025385ns) {DropIncomingHandshake:441(1100ns) DropOutgoingHandshake:456(86ns) SwitchToState:178(572ns) ProcessPendingSessionEvents:495(1243ns)}}
2025-11-11T17:20:54.978348Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607416:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.978378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519050868607416:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:54.980333Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519050868607416:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:54.980613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519046573640053:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:54.980678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:54.980731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519046573640053:2049] [node 3] ICP22 created new session: [2:7571519050868607443:2049]
2025-11-11T17:20:54.980778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519050868607443:2049] [node 3] ICS09 handshake done sender: [2:7571519050868607416:2111] self: [2:7571519050868607415:0] peer: [3:7571519051972057440:0] socket: 530 qp: -1
2025-11-11T17:20:54.980813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519050868607443:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:54.980852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:54.999706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519050212177141:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:54.999807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:54.999855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519050212177141:2048] [node 1] ICP22 created new session: [5:7571519054507144519:2048]
2025-11-11T17:20:54.999905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519054507144519:2048] [node 1] ICS09 handshake done sender: [5:7571519054507144511:2113] self: [5:7571519054507144512:0] peer: [1:7571519053157585050:0] socket: 541 qp: -1
2025-11-11T17:20:54.999943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519054507144519:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:55.000006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:54.839487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:54.839533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519048862617673:2048] [node 2] ICP22 created new session: [1:7571519053157585066:2048]
2025-11-11T17:20:54.839578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519053157585066:2048] [node 2] ICS09 handshake done sender: [1:7571519053157585047:2121] self: [1:7571519053157585046:0] peer: [2:7571519050868607428:0] socket: 538 qp: -1
2025-11-11T17:20:54.839618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519053157585066:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:54.839674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:54.840969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519053157585042:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:54.841023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:54.841067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519053157585042:2108] [node 6] ICP22 created new session: [1:7571519053157585068:2108]
2025-11-11T17:20:54.841116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519053157585068:2108] [node 6] ICS09 handshake done sender: [1:7571519053157585049:2122] self: [1:7571519053157585048:0] peer: [6:7571519053212537700:0] socket: 539 qp: -1
2025-11-11T17:20:54.841150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519053157585068:2108] [node 6] ICS10 traffic start
2025-11-11T17:20:54.841183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:54.851824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519048862617674:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(520203043ns) {Configure:104(520194841ns) {ProcessConfigured:135(75730732ns) {StartInitialHandshake:157(75727140ns) {DropHandshakes:469(19943366ns) {DropIncomingHandshake:441(1242ns) DropOutgoingHandshake:456(483ns)} GenerateSessionVirtualId:988(1375ns) PrepareNewSessionHandshake:476(4844ns) {SwitchToState:178(1992ns)}}}}}
2025-11-11T17:20:54.868266Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519053157585051:2123] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:54.882803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519053157585043:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:54.882891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:54.882936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519053157585043:2109] [node 5] ICP22 created new session: [1:7571519053157585071:2109]
2025-11-11T17:20:54.882980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519053157585071:2109] [node 5] ICS09 handshake done sender: [1:7571519053157585051:2123] self: [1:7571519053157585050:0] peer: [5:7571519054507144512:0] socket: 540 qp: -1
2025-11-11T17:20:54.883022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519053157585071:2109] [node 5] ICS10 traffic start
2025-11-11T17:20:54.883079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:54.946055Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585063:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.946083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519053157585063:2126] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:54.949652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519053157585056:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.949709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519053157585056:2124] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:54.953886Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519053157585063:2126] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:54.993975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519053157585042:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(38993795ns) {HandleHandshakeStatus:329(38991467ns) {DropIncomingHandshake:441(14098ns) DropOutgoingHandshake:456(455ns) SwitchToState:178(858ns) ProcessPendingSessionEvents:495(38660389ns) {ForwardSessionEventToSession:612(168596ns) ForwardSessionEventToSession:612(4381695ns) ForwardSessionEventToSession:612(5026ns) ForwardSessionEventToSession:612(11950038ns) ForwardSessionEventToSession:612(10177261ns) ForwardSessionEventToSession:612(3904ns) ForwardSessionEventToSession:612(5827728ns) ForwardSessionEventToSession:612(6184ns) ForwardSessionEventToSession:612(3804ns) ForwardSessionEventToSession:612(6106554ns)}}}
2025-11-11T17:20:54.997785Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519053157585056:2124] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:55.009848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519048862617674:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.973698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519048917570326:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519053212537698:2114] held: no
2025-11-11T17:20:55.017663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519054507144518:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.893680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057432:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.017738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:55.034610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519053157585041:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.034701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:55.034764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519053157585041:2107] [node 4] ICP22 created new session: [1:7571519057452552373:2107]
2025-11-11T17:20:55.034830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519057452552373:2107] [node 4] ICS09 handshake done sender: [1:7571519053157585056:2124] self: [1:7571519053157585055:0] peer: [4:7571519054154441031:0] socket: 552 qp: -1
2025-11-11T17:20:55.034879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519057452552373:2107] [node 4] ICS10 traffic start
2025-11-11T17:20:55.034924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:55.035717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519048862617674:2049] [node 3] ICP22 created new session: [1:7571519057452552372:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.038523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519054507144509:2082] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:55.038953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519058802111818:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.026729Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607431:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.011276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519054154441032:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2609
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.013959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519053212537641:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:55.050459Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519058802111818:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.050617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519058802111820:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:55.050793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519050212177142:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.069675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519057452552372:2049] [node 3] ICS09 handshake done sender: [1:7571519053157585063:2126] self: [1:7571519053157585060:0] peer: [3:7571519051972057441:0] socket: 534 qp: -1
2025-11-11T17:20:55.069717Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519058802111820:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.069927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519054507144514:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.069961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519054507144514:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:54.906059Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057429:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.013723Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519054154441029:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:55.022125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519049859473647:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.046823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519054154441014:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.046867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519054154441014:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:55.054373Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519054154441021:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.054403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519054154441021:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:55.055682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519054154441028:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.056038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519049859473649:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:55.056712Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519054154441021:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:55.057026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519054154441007:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.057099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:55.057154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519054154441007:2066] [node 5] ICP22 created new session: [4:7571519058449408331:2066]
2025-11-11T17:20:55.057200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519058449408331:2066] [node 5] ICS09 handshake done sender: [4:7571519054154441021:2115] self: [4:7571519054154441020:0] peer: [5:7571519058802111819:0] socket: 570 qp: -1
2025-11-11T17:20:55.057238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519058449408331:2066] [node 5] ICS10 traffic start
2025-11-11T17:20:55.057300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:55.077682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519057452552372:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:54.907970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519051972057438:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33079
2025-11-11T17:20:55.026777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519050868607431:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:55.014900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519053212537704:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:55.089725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:55.072550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519058802111822:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:55.015224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519057507505002:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:55.015327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519057507505003:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:55.026258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519057507505002:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.026398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519048917570327:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:55.028097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519057507505006:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:55.028799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519057507505002:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:55.028993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519048917570327:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.029094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:55.029139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519048917570327:2049] [node 2] ICP22 created new session: [6:7571519057507505007:2049]
2025-11-11T17:20:55.029194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519057507505007:2049] [node 2] ICS09 handshake done sender: [6:7571519057507505002:2119] self: [6:7571519057507505005:0] peer: [2:7571519050868607430:0] socket: 580 qp: -1
2025-11-11T17:20:55.029230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519057507505007:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:55.029268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:55.046312Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519057507505003:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.046471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519053212537644:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:55.072879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519048917570326:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(46982325ns) {HandleHandshakeStatus:395(46976859ns) {LogHandshakeFail:472(6718750ns) {UpdateErrorStateLog:544(9778ns)} DropOutgoingHandshake:456(4773ns) ProcessPendingSessionEvents:495(14889622ns) {DropSessionEvent:512(19658ns) DropSessionEvent:512(12743ns) DropSessionEvent:512(7833ns) DropSessionEvent:512(6058ns) DropSessionEvent:512(5677ns) DropSessionEvent:512(5776ns) DropSessionEvent:512(7185ns) DropSessionEvent:512(28712ns) DropSessionEvent:512(6754ns) DropSessionEvent:512(6256ns)}}}
2025-11-11T17:20:55.086309Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519053212537704:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.080040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.028278Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519050868607431:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:54.943569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519047677090061:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:54.943736Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057438:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.943832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057431:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:54.944104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519051972057430:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33079
2025-11-11T17:20:54.944286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519047677090060:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:54.980003Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057430:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.022192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519051972057396:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(192448133ns) {Configure:104(192433627ns) {ProcessConfigured:135(3787535ns) {StartInitialHandshake:157(3784272ns) {DropHandshakes:469(3373ns) {DropIncomingHandshake:441(869ns) DropOutgoingHandshake:456(615ns)} GenerateSessionVirtualId:988(694ns) PrepareNewSessionHandshake:476(8056ns) {SwitchToState:178(5718ns)}}}}}
2025-11-11T17:20:55.042114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519056267024741:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33079
2025-11-11T17:20:55.085714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519056267024742:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33079
2025-11-11T17:20:55.098501Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519051972057438:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:55.101646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057426:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.101687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519051972057426:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:55.077894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519054507144514:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.085998Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519054154441014:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:55.113680Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519051972057429:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:55.125776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519051972057426:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.134057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519047677090060:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.134169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:55.134221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519047677090060:2048] [node 1] ICP22 created new session: [3:7571519056267024745:2048]
2025-11-11T17:20:55.134267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519056267024745:2048] [node 1] ICS09 handshake done sender: [3:7571519051972057438:2119] self: [3:7571519051972057441:0] peer: [1:7571519053157585060:0] socket: 569 qp: -1
2025-11-11T17:20:55.134305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519056267024745:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:55.134339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:55.134813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057428:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.134837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519051972057428:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:55.094488Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519053212537698:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:55.065850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519050868607423:2105] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.147030Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519051972057428:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:55.147311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519051972057409:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.147405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:55.147453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519051972057409:2077] [node 4] ICP22 created new session: [3:7571519056267024749:2077]
2025-11-11T17:20:55.147519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519056267024749:2077] [node 4] ICS09 handshake done sender: [3:7571519051972057426:2111] self: [3:7571519051972057424:0] peer: [4:7571519058449408330:0] socket: 565 qp: -1
2025-11-11T17:20:55.147556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519056267024749:2077] [node 4] ICS10 traffic start
2025-11-11T17:20:55.147615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:55.148001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519051972057422:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.148078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:55.148117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519051972057422:2091] [node 6] ICP22 created new session: [3:7571519056267024751:2091]
2025-11-11T17:20:55.148169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519056267024751:2091] [node 6] ICS09 handshake done sender: [3:7571519051972057428:2112] self: [3:7571519051972057427:0] peer: [6:7571519057507505014:0] socket: 528 qp: -1
2025-11-11T17:20:55.148194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519056267024751:2091] [node 6] ICS10 traffic start
2025-11-11T17:20:55.148223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:55.152473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519047677090061:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.100194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519054507144510:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.089809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519049859473647:2048] [node 1] ICP22 created new session: [4:7571519058449408334:2048]
2025-11-11T17:20:55.065909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:55.101914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519048917570326:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.152579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:55.098009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519054154440989:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.098084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:55.100263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:55.101617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519058449408334:2048] [node 1] ICS09 handshake done sender: [4:7571519054154441029:2119] self: [4:7571519054154441031:0] peer: [1:7571519053157585055:0] socket: 568 qp: -1
2025-11-11T17:20:55.106568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519058449408334:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:55.165956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519048862617674:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(112654614ns) {HandleHandshakeStatus:329(112650763ns) {DropIncomingHandshake:441(643ns) DropOutgoingHandshake:456(667ns) SwitchToState:178(996ns) ProcessPendingSessionEvents:495(20634200ns) {ForwardSessionEventToSession:612(55873ns) ForwardSessionEventToSession:612(5749ns) ForwardSessionEventToSession:612(3275ns) ForwardSessionEventToSession:612(5763ns) ForwardSessionEventToSession:612(3037ns) ForwardSessionEventToSession:612(2514ns) ForwardSessionEventToSession:612(11858953ns) ForwardSessionEventToSession:612(4857ns) ForwardSessionEventToSession:612(3701ns) ForwardSessionEventToSession:612(8673436ns)}}}
2025-11-11T17:20:55.109689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519054154440989:2065] [node 6] ICP22 created new session: [4:7571519058449408335:2065]
2025-11-11T17:20:55.065969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519050868607423:2105] [node 6] ICP22 created new session: [2:7571519055163574742:2105]
2025-11-11T17:20:55.106037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519057507505010:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:55.152628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519047677090061:2049] [node 2] ICP22 created new session: [3:7571519056267024753:2049]
2025-11-11T17:20:55.066016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519055163574742:2105] [node 6] ICS09 handshake done sender: [2:7571519050868607431:2117] self: [2:7571519050868607430:0] peer: [6:7571519057507505005:0] socket: 577 qp: -1
2025-11-11T17:20:55.117786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:55.106618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:55.100312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519054507144510:2083] [node 6] ICP22 created new session: [5:7571519058802111825:2083]
2025-11-11T17:20:55.100361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519058802111825:2083] [node 6] ICS09 handshake done sender: [5:7571519054507144514:2114] self: [5:7571519054507144513:0] peer: [6:7571519057507505009:0] socket: 531 qp: -1
2025-11-11T17:20:55.117870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519048917570326:2048] [node 1] ICP22 created new session: [6:7571519057507505012:2048]
2025-11-11T17:20:55.117916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519057507505012:2048] [node 1] ICS09 handshake done sender: [6:7571519053212537698:2114] self: [6:7571519053212537700:0] peer: [1:7571519053157585048:0] socket: 551 qp: -1
2025-11-11T17:20:55.117964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519057507505012:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:55.118002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:55.118283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519048917570326:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16302782ns) {HandleHandshakeStatus:329(16299794ns) {DropIncomingHandshake:441(1389ns) DropOutgoingHandshake:456(467ns) SwitchToState:178(801ns) ProcessPendingSessionEvents:495(1282ns)}}
2025-11-11T17:20:55.118393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519048917570328:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:55.119104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519057507505011:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:55.130220Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:55.146019Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12554) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:55.149020Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519057507505003:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:55.149245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519053212537644:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519057507505003:2120] held: no
2025-11-11T17:20:55.149351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519053212537644:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.149409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:55.149452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519053212537644:2067] [node 5] ICP22 created new session: [6:7571519057507505015:2067]
2025-11-11T17:20:55.149497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519057507505015:2067] [node 5] ICS09 handshake done sender: [6:7571519057507505003:2120] self: [6:7571519057507505009:0] peer: [5:7571519054507144513:0] socket: 581 qp: -1
2025-11-11T17:20:55.149525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519057507505015:2067] [node 5] ICS10 traffic start
2025-11-11T17:20:55.149580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:55.150057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519053212537699:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:55.152672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519056267024753:2049] [node 2] ICS09 handshake done sender: [3:7571519051972057429:2113] self: [3:7571519051972057440:0] peer: [2:7571519050868607415:0] socket: 557 qp: -1
2025-11-11T17:20:55.153501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519053212537641:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12554) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519053212537699:2115] held: no
2025-11-11T17:20:55.152712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519056267024753:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:55.182166Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:55.152759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:55.185888Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519051972057444:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.185914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519051972057444:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:55.187391Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519051972057444:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:55.234212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519051972057396:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.249808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519047677090061:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(57218798ns) {HandleHandshakeStatus:329(57216058ns) {DropIncomingHandshake:441(874ns) DropOutgoingHandshake:456(6307ns) SwitchToState:178(496ns) ProcessPendingSessionEvents:495(56807294ns) {ForwardSessionEventToSession:612(4844707ns) ForwardSessionEventToSession:612(16207012ns) ForwardSessionEventToSession:612(4239ns) ForwardSessionEventToSession:612(35147ns) ForwardSessionEventToSession:612(3922968ns) ForwardSessionEventToSession:612(2947ns) ForwardSessionEventToSession:612(15998535ns) ForwardSessionEventToSession:612(12144ns) ForwardSessionEventToSession:612(3771ns) ForwardSessionEventToSession:612(7980390ns)}}}
2025-11-11T17:20:55.249992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:55.125671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519058449408335:2065] [node 6] ICS09 handshake done sender: [4:7571519054154441014:2113] self: [4:7571519054154441013:0] peer: [6:7571519057507505001:0] socket: 549 qp: -1
2025-11-11T17:20:55.126139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519058449408335:2065] [node 6] ICS10 traffic start
2025-11-11T17:20:55.137104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519058449408337:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2609
2025-11-11T17:20:55.137888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:55.178757Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519049859473647:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(156330524ns) {HandleHandshakeStatus:329(156327280ns) {DropIncomingHandshake:441(6997ns) DropOutgoingHandshake:456(15629ns) SwitchToState:178(885ns) ProcessPendingSessionEvents:495(16662436ns) {ForwardSessionEventToSession:612(16602151ns) ForwardSessionEventToSession:612(6790ns) ForwardSessionEventToSession:612(7842ns) ForwardSessionEventToSession:612(9150ns) ForwardSessionEventToSession:612(2951ns) ForwardSessionEventToSession:612(5269ns) ForwardSessionEventToSession:612(3197ns) ForwardSessionEventToSession:612(6108ns) ForwardSessionEventToSession:612(3739ns) ForwardSessionEventToSession:612(2992ns)}}}
2025-11-11T17:20:55.178956Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:55.203482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519049859473649:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519054154441028:2118] held: no
2025-11-11T17:20:55.204074Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519054154441028:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:55.183251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519053212537641:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.183351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:55.066062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519055163574742:2105] [node 6] ICS10 traffic start
2025-11-11T17:20:55.183398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519053212537641:2063] [node 4] ICP22 created new session: [6:7571519057507505018:2063]
2025-11-11T17:20:55.183450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519057507505018:2063] [node 4] ICS09 handshake done sender: [6:7571519053212537699:2115] self: [6:7571519057507505001:0] peer: [4:7571519054154441013:0] socket: 553 qp: -1
2025-11-11T17:20:55.066126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:55.066953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519050868607436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52264
2025-11-11T17:20:55.079109Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607429:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.079147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519050868607429:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:55.106428Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519050868607429:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:55.111660Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519050868607436:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.144175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519050868607408:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:55.162180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519046573640053:2049] [node 3] ICP18 handshake (actor [2:7571519050868607436:2119]) is too old
2025-11-11T17:20:55.181740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [2:7571519046573640053:2049] [node 3] ICP35 event processing took too much time StateWork:293(39916178ns) {IncomingHandshake:261(36209897ns)}
2025-11-11T17:20:55.201830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:55.201894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519050868607408:2107] [node 5] ICP22 created new session: [2:7571519055163574745:2107]
2025-11-11T17:20:55.217832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519055163574745:2107] [node 5] ICS09 handshake done sender: [2:7571519050868607429:2116] self: [2:7571519050868607425:0] peer: [5:7571519058802111821:0] socket: 576 qp: -1
2025-11-11T17:20:55.217892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519055163574745:2107] [node 5] ICS10 traffic start
2025-11-11T17:20:55.237887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:55.238448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519050868607408:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(100102261ns) {HandleHandshakeStatus:329(100099082ns) {DropIncomingHandshake:441(17754ns) DropOutgoingHandshake:456(572ns) SwitchToState:178(723ns) ProcessPendingSessionEvents:495(112026ns) {ForwardSessionEventToSession:612(58008ns) ForwardSessionEventToSession:612(7078ns) ForwardSessionEventToSession:612(3668ns) ForwardSessionEventToSession:612(5563ns) ForwardSessionEventToSession:612(9429ns) ForwardSessionEventToSession:612(3343ns) ForwardSessionEventToSession:612(3006ns) ForwardSessionEventToSession:612(5390ns) ForwardSessionEventToSession:612(2693ns) ForwardSessionEventToSession:612(5271ns)}}}
2025-11-11T17:20:55.206142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519054154440989:2065] [node 6] ICP35 event processing took too much time PendingConnection:278(107887354ns) {HandleHandshakeStatus:329(107884771ns) {DropIncomingHandshake:441(16972ns) DropOutgoingHandshake:456(883ns) SwitchToState:178(881ns) ProcessPendingSessionEvents:495(32033544ns) {ForwardSessionEventToSession:612(31969976ns) ForwardSessionEventToSession:612(5108ns) ForwardSessionEventToSession:612(6081ns) ForwardSessionEventToSession:612(2965ns) ForwardSessionEventToSession:612(3219ns) ForwardSessionEventToSession:612(5499ns) ForwardSessionEventToSession:612(2682ns) ForwardSessionEventToSession:612(2716ns) ForwardSessionEventToSession:612(18266ns) ForwardSessionEventToSession:612(7408ns)}}}
2025-11-11T17:20:55.218220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519049859473649:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.218327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:55.218378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519049859473649:2050] [node 3] ICP22 created new session: [4:7571519058449408339:2050]
2025-11-11T17:20:55.218426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519058449408339:2050] [node 3] ICS09 handshake done sender: [4:7571519054154441028:2118] self: [4:7571519058449408330:0] peer: [3:7571519051972057424:0] socket: 567 qp: -1
2025-11-11T17:20:55.218461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519058449408339:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:55.218521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:55.100399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519058802111825:2083] [node 6] ICS10 traffic start
2025-11-11T17:20:55.100439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:55.100965Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519054507144518:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:55.132021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519054507144509:2082] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.132100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:55.132150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519054507144509:2082] [node 4] ICP22 created new session: [5:7571519058802111828:2082]
2025-11-11T17:20:55.132217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519058802111828:2082] [node 4] ICS09 handshake done sender: [5:7571519054507144518:2117] self: [5:7571519058802111819:0] peer: [4:7571519054154441020:0] socket: 574 qp: -1
2025-11-11T17:20:55.132256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519058802111828:2082] [node 4] ICS10 traffic start
2025-11-11T17:20:55.132312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:55.133341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519058802111827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:55.133908Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519058802111818:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:55.142360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519050212177142:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.183506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519057507505018:2063] [node 4] ICS10 traffic start
2025-11-11T17:20:55.183564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:55.183937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519048917570328:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519053212537704:2118] held: no
2025-11-11T17:20:55.194012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519057507505017:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-11T17:20:55.242081Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519053212537704:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:55.254041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519048917570328:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.254125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:55.254182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519048917570328:2050] [node 3] ICP22 created new session: [6:7571519057507505020:2050]
2025-11-11T17:20:55.254226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519057507505020:2050] [node 3] ICS09 handshake done sender: [6:7571519053212537704:2118] self: [6:7571519057507505014:0] peer: [3:7571519051972057427:0] socket: 571 qp: -1
2025-11-11T17:20:55.254263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519057507505020:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:55.254314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:55.250045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519051972057396:2078] [node 5] ICP22 created new session: [3:7571519056267024756:2078]
2025-11-11T17:20:55.142496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:55.142570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519050212177142:2049] [node 2] ICP22 created new session: [5:7571519058802111831:2049]
2025-11-11T17:20:55.142622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519058802111831:2049] [node 2] ICS09 handshake done sender: [5:7571519058802111818:2118] self: [5:7571519058802111821:0] peer: [2:7571519050868607425:0] socket: 579 qp: -1
2025-11-11T17:20:55.142666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519058802111831:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:55.142726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:55.150318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519058802111833:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:55.166334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519058802111833:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:55.166470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519050212177143:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:55.187198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519058802111835:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24650
2025-11-11T17:20:55.187855Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519058802111833:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:55.188052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519050212177143:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:55.188108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:55.188156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519050212177143:2050] [node 3] ICP22 created new session: [5:7571519058802111836:2050]
2025-11-11T17:20:55.188218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519058802111836:2050] [node 3] ICS09 handshake done sender: [5:7571519058802111833:2122] self: [5:7571519058802111834:0] peer: [3:7571519051972057443:0] socket: 595 qp: -1
2025-11-11T17:20:55.188253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519058802111836:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:55.188286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:55.250119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519056267024756:2078] [node 5] ICS09 handshake done sender: [3:7571519051972057444:2120] self: [3:7571519051972057443:0] peer: [5:7571519058802111834:0] socket: 582 qp: -1
2025-11-11T17:20:55.250163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519056267024756:2078] [node 5] ICS10 traffic start
2025-11-11T17:20:55.250202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:55.250897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519051972057396:2078] [node 5] ICP35 event processing took too much time PendingConnection:278(36787889ns) {HandleHandshakeStatus:329(28975532ns) {DropIncomingHandshake:441(36642ns) DropOutgoingHandshake:456(874ns) SwitchToState:178(1118ns) ProcessPendingSessionEvents:495(295123ns) {ForwardSessionEventToSession:612(241334ns) ForwardSessionEventToSession:612(5017ns) ForwardSessionEventToSession:612(11250ns) ForwardSessionEventToSession:612(3295ns) ForwardSessionEventToSession:612(6290ns) ForwardSessionEventToSession:612(5572ns) ForwardSessionEventToSession:612(3052ns) ForwardSessionEventToSession:612(5482ns) ForwardSessionEventToSession:612(2315ns) ForwardSessionEventToSession:612(2778ns)}}}
2025-11-11T17:20:55.327304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:55.327555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519050868607435:2048] [node 1] ICS01 socket: 542 reason# EndOfStream
2025-11-11T17:20:55.327593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519046573640052:2048] [node 1] ICP30 unregister session Session# [2:7571519050868607435:2048] VirtualId# [2:7571519050868607428:0]
2025-11-11T17:20:55.327627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519050868607435:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.327726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:55.327769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:55.343182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:55.343422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519056267024745:2048] [node 1] ICS01 socket: 569 reason# EndOfStream
2025-11-11T17:20:55.343471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519047677090060:2048] [node 1] ICP30 unregister session Session# [3:7571519056267024745:2048] VirtualId# [3:7571519051972057441:0]
2025-11-11T17:20:55.343509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519056267024745:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.343603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:55.343649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:55.357773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:55.358105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519054507144519:2048] [node 1] ICS01 socket: 541 reason# EndOfStream
2025-11-11T17:20:55.358181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519050212177141:2048] [node 1] ICP30 unregister session Session# [5:7571519054507144519:2048] VirtualId# [5:7571519054507144512:0]
2025-11-11T17:20:55.358231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519054507144519:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.358010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:55.358151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:55.358442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519058449408334:2048] [node 1] ICS07 socket disconnect 568 reason# EndOfStream
2025-11-11T17:20:55.358481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519058449408334:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.358579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:55.358642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519058449408334:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:55.358362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:55.358426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:55.358308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519057507505012:2048] [node 1] ICS01 socket: 551 reason# EndOfStream
2025-11-11T17:20:55.358355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519048917570326:2048] [node 1] ICP30 unregister session Session# [6:7571519057507505012:2048] VirtualId# [6:7571519053212537700:0]
2025-11-11T17:20:55.358394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519057507505012:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.358506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:55.358566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:55.361837Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Couldn't connect to any resolved address
2025-11-11T17:20:55.362457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519049859473647:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:55.362516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519058449408334:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:55.362551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519049859473647:2048] [node 1] ICP30 unregister session Session# [4:7571519058449408334:2048] VirtualId# [4:7571519054154441031:0]
2025-11-11T17:20:55.362612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:55.362656Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519049859473647:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:55.362671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3800) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:55.373885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519049859473647:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:55.429840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:55.430067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519056267024753:2049] [node 2] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-11T17:20:55.430095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519056267024753:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.430216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:55.430335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519056267024753:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:55.441734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:55.441949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519058802111831:2049] [node 2] ICS01 socket: 579 reason# EndOfStream
2025-11-11T17:20:55.441992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519050212177142:2049] [node 2] ICP30 unregister session Session# [5:7571519058802111831:2049] VirtualId# [5:7571519058802111821:0]
2025-11-11T17:20:55.442025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519058802111831:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.442101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:55.442146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:55.443200Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10444) Couldn't connect to any resolved address
2025-11-11T17:20:55.443355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519047677090061:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10444) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:55.446318Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:55.446584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519057507505007:2049] [node 2] ICS01 socket: 580 reason# EndOfStream
2025-11-11T17:20:55.446629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519048917570327:2049] [node 2] ICP30 unregister session Session# [6:7571519057507505007:2049] VirtualId# [6:7571519057507505005:0]
2025-11-11T17:20:55.446662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519057507505007:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.446765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:55.446813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:55.453803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:55.454124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519054154441026:2049] [node 2] ICS01 socket: 532 reason# EndOfStream
2025-11-11T17:20:55.454181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519049859473648:2049] [node 2] ICP30 unregister session Session# [4:7571519054154441026:2049] VirtualId# [4:7571519054154441022:0]
2025-11-11T17:20:55.454222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519054154441026:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.454350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:55.454398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:55.457754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519056267024753:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:55.457822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519047677090061:2049] [node 2] ICP30 unregister session Session# [3:7571519056267024753:2049] VirtualId# [3:7571519051972057440:0]
2025-11-11T17:20:55.457881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:55.457921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519047677090061:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:55.457938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:55.485893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519047677090061:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:55.579598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:55.579865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519058449408339:2050] [node 3] ICS01 socket: 567 reason# EndOfStream
2025-11-11T17:20:55.579904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519049859473649:2050] [node 3] ICP30 unregister session Session# [4:7571519058449408339:2050] VirtualId# [4:7571519058449408330:0]
2025-11-11T17:20:55.579945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519058449408339:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.580032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:55.580074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:55.589727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:55.590011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519058802111836:2050] [node 3] ICS01 socket: 595 reason# EndOfStream
2025-11-11T17:20:55.590050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519050212177143:2050] [node 3] ICP30 unregister session Session# [5:7571519058802111836:2050] VirtualId# [5:7571519058802111834:0]
2025-11-11T17:20:55.590084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519058802111836:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.590199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:55.590243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:55.591329Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:55.591545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519057507505020:2050] [node 3] ICS01 socket: 571 reason# EndOfStream
2025-11-11T17:20:55.591587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519048917570328:2050] [node 3] ICP30 unregister session Session# [6:7571519057507505020:2050] VirtualId# [6:7571519057507505014:0]
2025-11-11T17:20:55.591627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519057507505020:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.591748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:55.591791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:55.645759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:55.645759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:55.646011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519058802111828:2082] [node 4] ICS01 socket: 574 reason# EndOfStream
2025-11-11T17:20:55.646047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519054507144509:2082] [node 4] ICP30 unregister session Session# [5:7571519058802111828:2082] VirtualId# [5:7571519058802111819:0]
2025-11-11T17:20:55.646082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519058802111828:2082] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.646170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:55.646214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:55.646240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519057507505018:2063] [node 4] ICS01 socket: 553 reason# EndOfStream
2025-11-11T17:20:55.646287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519053212537641:2063] [node 4] ICP30 unregister session Session# [6:7571519057507505018:2063] VirtualId# [6:7571519057507505001:0]
2025-11-11T17:20:55.646314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519057507505018:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.646387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:55.646432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:55.726061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:55.726406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519057507505015:2067] [node 5] ICS01 socket: 581 reason# EndOfStream
2025-11-11T17:20:55.726482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519053212537644:2067] [node 5] ICP30 unregister session Session# [6:7571519057507505015:2067] VirtualId# [6:7571519057507505009:0]
2025-11-11T17:20:55.726549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519057507505015:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:55.726719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:55.726797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:55.833724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519056235383493:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:55.833783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519056235383494:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:55.848047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519055728800627:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:55.848087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519055728800628:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:55.878051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519055175111071:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:55.878093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519055175111072:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:55.973937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519058987930952:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:55.973986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519058987930953:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:55.974011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519058987930954:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:56.038004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519062495928106:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:56.038055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519062495928107:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:56.038083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519062495928108:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:56.254030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519059804413645:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:56.254071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519059804413646:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:56.254097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519059804413647:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.304746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519060023767950:2058] [node 4] ICP01 ready to work
2025-11-11T17:20:56.298030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898278:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
2025-11-11T17:20:56.260535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519059470078384:2058] [node 6] ICP01 ready to work
2025-11-11T17:20:56.242364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519060530350804:2060] [node 4] ICP01 ready to work
2025-11-11T17:20:56.347353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519062495928129:2066] [node 6] ICP01 ready to work
2025-11-11T17:20:56.347656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519062495928130:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.344377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519060530350860:2068] [node 5] ICP01 ready to work
2025-11-11T17:20:56.353721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519059470078429:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.377713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898294:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
2025-11-11T17:20:56.352262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519060023767986:2061] [node 6] ICP01 ready to work
2025-11-11T17:20:56.429666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519059804413681:2060] [node 4] ICP01 ready to work
2025-11-11T17:20:56.501642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519063282898300:2062] [node 5] ICP01 ready to work
2025-11-11T17:20:56.371350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519060530350859:2074] [node 6] ICP01 ready to work
2025-11-11T17:20:56.499122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519062495928108:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(290057693ns) {Configure:104(290051479ns) {ProcessConfigured:135(56885891ns) {StartInitialHandshake:157(56881130ns) {DropHandshakes:469(3738ns) {DropIncomingHandshake:441(864ns) DropOutgoingHandshake:456(665ns)} GenerateSessionVirtualId:988(952ns) PrepareNewSessionHandshake:476(4714ns) {SwitchToState:178(2254ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.495957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519055175111072:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(229985455ns) {Configure:104(229979211ns) {ProcessConfigured:135(69379ns) {StartInitialHandshake:157(65797ns) {DropHandshakes:469(3838ns) {DropIncomingHandshake:441(714ns) DropOutgoingHandshake:456(580ns)} GenerateSessionVirtualId:988(998ns) PrepareNewSessionHandshake:476(2379ns) {SwitchToState:178(770ns)}}}}}
2025-11-11T17:20:56.506160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519060023767988:2070] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.525732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519059804413698:2065] [node 5] ICP01 ready to work
2025-11-11T17:20:56.434068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519060530350861:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23341
2025-11-11T17:20:56.516361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519059470078432:2072] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.577668Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519063282898278:2075] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.601244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519063282898322:2084] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.540048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519059804413646:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25882464ns) {Configure:104(25875616ns) {ProcessConfigured:135(62680ns) {StartInitialHandshake:157(59831ns) {DropHandshakes:469(3738ns) {DropIncomingHandshake:441(746ns) DropOutgoingHandshake:456(585ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:476(2475ns) {SwitchToState:178(836ns)}}}}}
2025-11-11T17:20:56.587122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928184:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.597979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519060023767988:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(89307627ns) {Configure:104(89302581ns) {ProcessConfigured:135(19900575ns) {StartInitialHandshake:157(19895623ns) {DropHandshakes:469(3465ns) {DropIncomingHandshake:441(789ns) DropOutgoingHandshake:456(531ns)} GenerateSessionVirtualId:988(783ns) PrepareNewSessionHandshake:476(5774ns) {SwitchToState:178(2532ns)}}}}}
2025-11-11T17:20:56.578455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519059470078441:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-11T17:20:56.499582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519056235383494:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(277190269ns) {Configure:104(277184011ns) {ProcessConfigured:135(57349640ns) {StartInitialHandshake:157(57347423ns) {DropHandshakes:469(723ns) {DropIncomingHandshake:441(109ns) DropOutgoingHandshake:456(89ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:476(3803ns) {SwitchToState:178(1386ns)}}}}}
2025-11-11T17:20:56.602984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928183:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.499745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350861:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.616271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519058987930952:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.578655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519059470078442:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.544105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413714:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.600686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519060023767990:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44649
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.614003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519060530350866:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23341
2025-11-11T17:20:56.616598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898324:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
2025-11-11T17:20:56.642167Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519062495928183:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.601363Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519059470078442:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.602041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023767990:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.602145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519060023767997:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44649
2025-11-11T17:20:56.602204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519060023767998:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44649
2025-11-11T17:20:56.630228Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023767998:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.658250Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023767997:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.677858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519060023767989:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44649
2025-11-11T17:20:56.603922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519059804413714:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.703398Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023767992:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.616711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898325:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
2025-11-11T17:20:56.647025Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350806:2062] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.646528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928185:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.637714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519059470078441:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.662711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519055175111071:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(431212132ns) {Configure:104(431205329ns) {ProcessConfigured:135(24599624ns) {StartInitialHandshake:157(24596549ns) {DropHandshakes:469(3671ns) {DropIncomingHandshake:441(929ns) DropOutgoingHandshake:456(582ns)} GenerateSessionVirtualId:988(1109ns) PrepareNewSessionHandshake:476(3290ns) {SwitchToState:178(1252ns)}}}}}
2025-11-11T17:20:56.663188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519055175111071:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.689975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519059470078429:2063] [node 4] ICP35 event processing took too much time PendingConnection:278(20121322ns) {IncomingHandshake:261(40525ns) {DropIncomingHandshake:441(286ns) PrepareNewSessionHandshake:476(3012ns) {SwitchToState:178(1266ns)} GenerateSessionVirtualId:988(211ns) IssueIncomingHandshakeReply:193(5939ns)}}
2025-11-11T17:20:56.700078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519059470078384:2058] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(426195374ns) {Configure:104(426182347ns) {ProcessConfigured:135(88115ns) {StartInitialHandshake:157(84405ns) {DropHandshakes:469(3312ns) {DropIncomingHandshake:441(607ns) DropOutgoingHandshake:456(730ns)} GenerateSessionVirtualId:988(1061ns) PrepareNewSessionHandshake:476(2988ns) {SwitchToState:178(738ns)}}}}}
2025-11-11T17:20:56.702652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519059470078450:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-11T17:20:56.703447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519060023767992:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.729844Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519059470078438:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.729909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519059470078438:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.625834Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519063282898324:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.611729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519059804413645:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(73601667ns) {Configure:104(73593974ns) {ProcessConfigured:135(82950ns) {StartInitialHandshake:157(79314ns) {DropHandshakes:469(3391ns) {DropIncomingHandshake:441(622ns) DropOutgoingHandshake:456(468ns)} GenerateSessionVirtualId:988(12518ns) PrepareNewSessionHandshake:476(2360ns) {SwitchToState:178(1003ns)}}}}}
2025-11-11T17:20:56.704033Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023767994:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.704055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519060023767994:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.713988Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519060023767992:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:56.714049Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519060023767994:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:56.732001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519060023767950:2058] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(404535506ns) {Configure:104(404530415ns) {ProcessConfigured:135(26551935ns) {StartInitialHandshake:157(18006743ns) {DropHandshakes:469(4274ns) {DropIncomingHandshake:441(740ns) DropOutgoingHandshake:456(950ns)} GenerateSessionVirtualId:988(2660ns) PrepareNewSessionHandshake:476(4365ns) {SwitchToState:178(1797ns)}}}}}
2025-11-11T17:20:56.681701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519062495928132:2069] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.681751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519062495928132:2069] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.689644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928182:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.689760Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519062495928185:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.689969Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519062495928184:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.694173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519062495928108:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.713829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519062495928107:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.717011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.721799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519062495928132:2069] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:56.733715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519062495928107:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(23774608ns) {IncomingHandshake:261(23770171ns) {DropIncomingHandshake:441(393ns) PrepareNewSessionHandshake:476(1169ns) {SwitchToState:178(355ns)} GenerateSessionVirtualId:988(323ns) IssueIncomingHandshakeReply:193(16037457ns) {SwitchToState:178(4474ns)}}}
2025-11-11T17:20:56.647064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519060530350806:2062] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.739514Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519059470078438:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:56.612482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413722:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:56.627234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519059804413698:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.654832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519059804413647:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(313808031ns) {Configure:104(313801845ns) {ProcessConfigured:135(114328ns) {StartInitialHandshake:157(111428ns) {DropHandshakes:469(3787ns) {DropIncomingHandshake:441(753ns) DropOutgoingHandshake:456(1080ns)} GenerateSessionVirtualId:988(922ns) PrepareNewSessionHandshake:476(3088ns) {SwitchToState:178(1046ns)}}}}}
2025-11-11T17:20:56.678641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519059804413722:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.697826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519059804413646:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.729726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519059804413646:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19007862ns) {IncomingHandshake:261(19005591ns) {DropIncomingHandshake:441(545ns) PrepareNewSessionHandshake:476(7396ns) {SwitchToState:178(4569ns)} GenerateSessionVirtualId:988(724ns) IssueIncomingHandshakeReply:193(60825ns) {SwitchToState:178(732ns)}}}
2025-11-11T17:20:56.731577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413728:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:56.731880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413727:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.738894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519062495928106:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(455294367ns) {Configure:104(455288396ns) {ProcessConfigured:135(12035742ns) {StartInitialHandshake:157(12031873ns) {DropHandshakes:469(4206ns) {DropIncomingHandshake:441(776ns) DropOutgoingHandshake:456(824ns)} GenerateSessionVirtualId:988(1086ns) PrepareNewSessionHandshake:476(4527ns) {SwitchToState:178(1477ns)}}}}}
2025-11-11T17:20:56.637717Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519063282898294:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.663790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898331:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
2025-11-11T17:20:56.739298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519058987930953:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(460832756ns) {Configure:104(460826536ns) {ProcessConfigured:135(16726647ns) {StartInitialHandshake:157(16723214ns) {DropHandshakes:469(3448ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(737ns)} GenerateSessionVirtualId:988(1005ns) PrepareNewSessionHandshake:476(4546ns) {SwitchToState:178(1170ns)}}}}}
2025-11-11T17:20:56.742144Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519063282898278:2075] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:56.748354Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:56.753789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519058987930952:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519063282898278:2075] held: no
2025-11-11T17:20:56.755759Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519063282898325:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.755939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519058987930954:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.658706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350866:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.791530Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519063282898320:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.746411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519055728800628:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(408539410ns) {Configure:104(408527884ns) {ProcessConfigured:135(7062201ns) {StartInitialHandshake:157(7059202ns) {DropHandshakes:469(5650ns) {DropIncomingHandshake:441(663ns) DropOutgoingHandshake:456(1671ns)} GenerateSessionVirtualId:988(1606ns) PrepareNewSessionHandshake:476(4066ns) {SwitchToState:178(1949ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.791570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519063282898320:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.754384Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519062495928182:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.746568Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023767989:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.663988Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519060530350806:2062] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.834145Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519059470078442:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:56.664403Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350863:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.792422Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519063282898320:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:56.818104Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519059804413722:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:56.837862Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:56.839732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519059804413646:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.839783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:56.839828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519059804413646:2049] [node 2] ICP22 created new session: [6:7571519059804413730:2049]
2025-11-11T17:20:56.839878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519059804413730:2049] [node 2] ICS09 handshake done sender: [6:7571519059804413722:2115] self: [6:7571519059804413726:0] peer: [2:7571519060023767991:0] socket: 567 qp: -1
2025-11-11T17:20:56.839920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519059804413730:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:56.839955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:56.840199Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519059804413646:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:56.840660Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519059804413714:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:56.849828Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:56.874148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413729:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:56.874668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413732:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:56.874716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413733:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:56.875491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519059804413698:2065] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519059804413714:2112] held: no
2025-11-11T17:20:56.754435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519062495928129:2066] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.761821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519060023767986:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.777931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519060023767988:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.778003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.792758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519063282898300:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.853913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519059470078436:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.664428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519060530350863:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.754502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:56.778062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519060023767988:2070] [node 5] ICP22 created new session: [2:7571519060023768011:2070]
2025-11-11T17:20:56.778122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519060023768011:2070] [node 5] ICS09 handshake done sender: [2:7571519060023767994:2113] self: [2:7571519060023767993:0] peer: [5:7571519062495928188:0] socket: 564 qp: -1
2025-11-11T17:20:56.778163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519060023768011:2070] [node 5] ICS10 traffic start
2025-11-11T17:20:56.778203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:56.779183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519055728800627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.789833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:56.789899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519060023767986:2061] [node 6] ICP22 created new session: [2:7571519060023768014:2061]
2025-11-11T17:20:56.789944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519060023768014:2061] [node 6] ICS09 handshake done sender: [2:7571519060023767992:2112] self: [2:7571519060023767991:0] peer: [6:7571519059804413726:0] socket: 563 qp: -1
2025-11-11T17:20:56.789991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519060023768014:2061] [node 6] ICS10 traffic start
2025-11-11T17:20:56.790034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:56.790572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519060023767986:2061] [node 6] ICP35 event processing took too much time PendingConnection:278(28528966ns) {HandleHandshakeStatus:329(28526595ns) {DropIncomingHandshake:441(15457ns) DropOutgoingHandshake:456(699ns) SwitchToState:178(529ns) ProcessPendingSessionEvents:495(107664ns) {ForwardSessionEventToSession:612(60795ns) ForwardSessionEventToSession:612(5770ns) ForwardSessionEventToSession:612(2927ns) ForwardSessionEventToSession:612(2592ns) ForwardSessionEventToSession:612(5029ns) ForwardSessionEventToSession:612(6770ns) ForwardSessionEventToSession:612(3043ns) ForwardSessionEventToSession:612(4426ns) ForwardSessionEventToSession:612(2381ns) ForwardSessionEventToSession:612(5399ns)}}}
2025-11-11T17:20:56.817427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519060023768017:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44649
2025-11-11T17:20:56.858125Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023768017:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.863156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519060023768019:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44649
2025-11-11T17:20:56.870414Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023768007:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.870439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519060023768007:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.871919Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519060023768007:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:56.872020Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519060023767989:2110] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:56.872228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519055728800627:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.872329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:56.872378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519055728800627:2048] [node 1] ICP22 created new session: [2:7571519060023768021:2048]
2025-11-11T17:20:56.872426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519060023768021:2048] [node 1] ICS09 handshake done sender: [2:7571519060023767989:2110] self: [2:7571519060023768013:0] peer: [1:7571519060530350814:0] socket: 545 qp: -1
2025-11-11T17:20:56.872462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519060023768021:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:56.872498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:56.887284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519060023767950:2058] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.887362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:56.887413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519060023767950:2058] [node 4] ICP22 created new session: [2:7571519060023768023:2058]
2025-11-11T17:20:56.887487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519060023768023:2058] [node 4] ICS09 handshake done sender: [2:7571519060023768007:2117] self: [2:7571519060023768006:0] peer: [4:7571519063282898341:0] socket: 546 qp: -1
2025-11-11T17:20:56.887524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519060023768023:2058] [node 4] ICS10 traffic start
2025-11-11T17:20:56.887583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:56.853963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519059470078436:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.856978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519059470078432:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.857058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:56.857106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519059470078432:2072] [node 5] ICP22 created new session: [3:7571519059470078453:2072]
2025-11-11T17:20:56.857178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519059470078453:2072] [node 5] ICS09 handshake done sender: [3:7571519059470078438:2112] self: [3:7571519059470078437:0] peer: [5:7571519062495928189:0] socket: 553 qp: -1
2025-11-11T17:20:56.857220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519059470078453:2072] [node 5] ICS10 traffic start
2025-11-11T17:20:56.857254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:56.857837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519055175111071:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.857906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:56.857955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519055175111071:2048] [node 1] ICP22 created new session: [3:7571519059470078455:2048]
2025-11-11T17:20:56.858017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519059470078455:2048] [node 1] ICS09 handshake done sender: [3:7571519059470078442:2114] self: [3:7571519059470078446:0] peer: [1:7571519060530350862:0] socket: 559 qp: -1
2025-11-11T17:20:56.858051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519059470078455:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:56.858087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:56.860918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519059470078452:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-11T17:20:56.861094Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:56.865884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519059470078432:2072] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:56.910467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519055175111071:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(52345211ns) {HandleHandshakeStatus:329(52340164ns) {DropIncomingHandshake:441(701ns) DropOutgoingHandshake:456(3943ns) SwitchToState:178(400ns) ProcessPendingSessionEvents:495(14416776ns) {ForwardSessionEventToSession:612(14355749ns) ForwardSessionEventToSession:612(7647ns) ForwardSessionEventToSession:612(3010ns) ForwardSessionEventToSession:612(4935ns) ForwardSessionEventToSession:612(2879ns) ForwardSessionEventToSession:612(3321ns) ForwardSessionEventToSession:612(8125ns) ForwardSessionEventToSession:612(3249ns) ForwardSessionEventToSession:612(12852ns) ForwardSessionEventToSession:612(2655ns)}}}
2025-11-11T17:20:56.911320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519059470078459:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.906863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519059804413729:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.665078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519060530350804:2060] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.792814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:56.754554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519062495928129:2066] [node 6] ICP22 created new session: [5:7571519062495928194:2066]
2025-11-11T17:20:56.754607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519062495928194:2066] [node 6] ICS09 handshake done sender: [5:7571519062495928132:2069] self: [5:7571519062495928131:0] peer: [6:7571519059804413723:0] socket: 540 qp: -1
2025-11-11T17:20:56.754643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519062495928194:2066] [node 6] ICS10 traffic start
2025-11-11T17:20:56.754682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:56.761689Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519062495928185:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:56.763708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.763831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519062495928130:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.770056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928198:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.801850Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519062495928193:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.802023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519062495928107:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.802081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:56.802132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519062495928107:2049] [node 2] ICP22 created new session: [5:7571519062495928200:2049]
2025-11-11T17:20:56.802179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519062495928200:2049] [node 2] ICS09 handshake done sender: [5:7571519062495928185:2117] self: [5:7571519062495928188:0] peer: [2:7571519060023767993:0] socket: 569 qp: -1
2025-11-11T17:20:56.802214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519062495928200:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:56.802249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:56.803141Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519062495928184:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:56.817027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928199:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.817544Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:56.821831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519062495928108:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.821903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:56.821962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519062495928108:2050] [node 3] ICP22 created new session: [5:7571519062495928202:2050]
2025-11-11T17:20:56.822012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519062495928202:2050] [node 3] ICS09 handshake done sender: [5:7571519062495928184:2116] self: [5:7571519062495928189:0] peer: [3:7571519059470078437:0] socket: 558 qp: -1
2025-11-11T17:20:56.822053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519062495928202:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:56.822088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:56.822567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519062495928106:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.822632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519062495928107:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:56.822770Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:56.839041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519062495928130:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519062495928182:2114] held: no
2025-11-11T17:20:56.839355Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519062495928182:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:56.855049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519062495928130:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.855102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:56.855151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519062495928130:2068] [node 4] ICP22 created new session: [5:7571519062495928206:2068]
2025-11-11T17:20:56.855196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519062495928206:2068] [node 4] ICS09 handshake done sender: [5:7571519062495928182:2114] self: [5:7571519062495928197:0] peer: [4:7571519063282898319:0] socket: 554 qp: -1
2025-11-11T17:20:56.855236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519062495928206:2068] [node 4] ICS10 traffic start
2025-11-11T17:20:56.855300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:56.856083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519062495928205:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41037
2025-11-11T17:20:56.895728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519062495928106:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20244229ns)
2025-11-11T17:20:56.914161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519062495928193:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:56.953937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519059470078436:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:56.924455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519059804413681:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.665168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:56.792851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519063282898300:2062] [node 5] ICP22 created new session: [4:7571519063282898337:2062]
2025-11-11T17:20:56.969703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519062495928106:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.969796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:56.986398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519062495928106:2048] [node 1] ICP22 created new session: [5:7571519062495928209:2048]
2025-11-11T17:20:56.963541Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519059470078449:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.926053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519059804413733:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:56.665215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519060530350804:2060] [node 4] ICP22 created new session: [1:7571519060530350869:2060]
2025-11-11T17:20:56.963584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519059470078449:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.946207Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519059804413732:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.792894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519063282898337:2062] [node 5] ICS09 handshake done sender: [4:7571519063282898320:2113] self: [4:7571519063282898319:0] peer: [5:7571519062495928197:0] socket: 548 qp: -1
2025-11-11T17:20:57.001679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519062495928209:2048] [node 1] ICS09 handshake done sender: [5:7571519062495928193:2120] self: [5:7571519062495928204:0] peer: [1:7571519060530350872:0] socket: 581 qp: -1
2025-11-11T17:20:56.665268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519060530350869:2060] [node 4] ICS09 handshake done sender: [1:7571519060530350806:2062] self: [1:7571519060530350805:0] peer: [4:7571519063282898328:0] socket: 530 qp: -1
2025-11-11T17:20:56.946666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519059804413645:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.946761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413737:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:56.953882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519059804413647:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.961826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519059804413698:2065] [node 5] ICP35 event processing took too much time PendingConnection:278(31365930ns) {HandleHandshakeStatus:395(31361509ns) {LogHandshakeFail:472(16436ns) {UpdateErrorStateLog:544(4964ns)} DropOutgoingHandshake:456(1938ns) ProcessPendingSessionEvents:495(26242637ns) {DropSessionEvent:512(34573ns) DropSessionEvent:512(5388ns) DropSessionEvent:512(5865ns) DropSessionEvent:512(6202ns) DropSessionEvent:512(5093ns) DropSessionEvent:512(5382ns) DropSessionEvent:512(4062ns) DropSessionEvent:512(3905ns) DropSessionEvent:512(5477ns) DropSessionEvent:512(3729ns)}}}
2025-11-11T17:20:56.962149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519059804413698:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.970127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:56.970188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519059804413698:2065] [node 5] ICP22 created new session: [6:7571519059804413740:2065]
2025-11-11T17:20:56.970234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519059804413740:2065] [node 5] ICS09 handshake done sender: [6:7571519059804413714:2112] self: [6:7571519059804413723:0] peer: [5:7571519062495928131:0] socket: 547 qp: -1
2025-11-11T17:20:56.970270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519059804413740:2065] [node 5] ICS10 traffic start
2025-11-11T17:20:56.970312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:56.971107Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519059804413729:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:56.978014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519059804413647:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(27728149ns) {IncomingHandshake:261(7930552ns) {DropIncomingHandshake:441(1490ns) PrepareNewSessionHandshake:476(4525ns) {SwitchToState:178(2874ns)} GenerateSessionVirtualId:988(1593ns) IssueIncomingHandshakeReply:193(66380ns) {SwitchToState:178(174ns)}}}
2025-11-11T17:20:56.978210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519059804413681:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.978301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:56.978360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519059804413681:2060] [node 4] ICP22 created new session: [6:7571519059804413743:2060]
2025-11-11T17:20:56.978402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519059804413743:2060] [node 4] ICS09 handshake done sender: [6:7571519059804413729:2119] self: [6:7571519059804413736:0] peer: [4:7571519063282898326:0] socket: 542 qp: -1
2025-11-11T17:20:56.978434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519059804413743:2060] [node 4] ICS10 traffic start
2025-11-11T17:20:56.978471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:56.978977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519059804413742:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:57.002822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519059804413733:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:57.002957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519064099381041:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56108
2025-11-11T17:20:57.019060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519059804413647:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:57.019172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:57.019219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519059804413647:2050] [node 3] ICP22 created new session: [6:7571519064099381042:2050]
2025-11-11T17:20:57.019269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519064099381042:2050] [node 3] ICS09 handshake done sender: [6:7571519059804413733:2121] self: [6:7571519059804413739:0] peer: [3:7571519059470078448:0] socket: 593 qp: -1
2025-11-11T17:20:57.019301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519064099381042:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:57.019338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:56.964356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519059470078429:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:57.017698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519062495928209:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:56.792924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519063282898337:2062] [node 5] ICS10 traffic start
2025-11-11T17:20:56.665303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519060530350869:2060] [node 4] ICS10 traffic start
2025-11-11T17:20:57.042160Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519059804413732:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:57.017760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:56.964421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:57.038923Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519060023768009:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.964471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519059470078429:2063] [node 4] ICP22 created new session: [3:7571519059470078461:2063]
2025-11-11T17:20:56.964517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519059470078461:2063] [node 4] ICS09 handshake done sender: [3:7571519059470078436:2111] self: [3:7571519059470078435:0] peer: [4:7571519063282898334:0] socket: 560 qp: -1
2025-11-11T17:20:56.964549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519059470078461:2063] [node 4] ICS10 traffic start
2025-11-11T17:20:56.964593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:56.969860Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519059470078459:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.982098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519059470078464:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-11T17:20:56.792959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:57.038970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519060023768009:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.982496Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519059470078449:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:57.003180Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519059470078464:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:57.009913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519055175111072:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:57.014717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519059470078384:2058] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:57.034204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:57.038892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519059470078384:2058] [node 6] ICP22 created new session: [3:7571519063765045763:2058]
2025-11-11T17:20:57.049809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519063765045763:2058] [node 6] ICS09 handshake done sender: [3:7571519059470078449:2116] self: [3:7571519059470078448:0] peer: [6:7571519059804413739:0] socket: 589 qp: -1
2025-11-11T17:20:57.057745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519063765045763:2058] [node 6] ICS10 traffic start
2025-11-11T17:20:57.061756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:56.665360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:56.717364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519060530350860:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(371173269ns) {Configure:104(371163809ns) {ProcessConfigured:135(19421004ns) {StartInitialHandshake:157(19417564ns) {DropHandshakes:469(3316ns) {DropIncomingHandshake:441(712ns) DropOutgoingHandshake:456(770ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:476(4264ns) {SwitchToState:178(1562ns)}}}}}
2025-11-11T17:20:56.726277Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519060530350863:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:56.774662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519056235383494:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.774745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:56.774801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519056235383494:2049] [node 3] ICP22 created new session: [1:7571519060530350876:2049]
2025-11-11T17:20:56.774847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519060530350876:2049] [node 3] ICS09 handshake done sender: [1:7571519060530350863:2113] self: [1:7571519060530350862:0] peer: [3:7571519059470078446:0] socket: 544 qp: -1
2025-11-11T17:20:56.774887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519060530350876:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:56.774950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:56.840890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350873:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.840918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519060530350873:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.842091Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519060530350873:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:56.842361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519060530350860:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.842436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:56.842479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519060530350860:2068] [node 5] ICP22 created new session: [1:7571519060530350880:2068]
2025-11-11T17:20:56.842525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519060530350880:2068] [node 5] ICS09 handshake done sender: [1:7571519060530350873:2115] self: [1:7571519060530350872:0] peer: [5:7571519062495928204:0] socket: 538 qp: -1
2025-11-11T17:20:56.842559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519060530350880:2068] [node 5] ICS10 traffic start
2025-11-11T17:20:56.842591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:56.850808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350815:2069] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.850854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519060530350815:2069] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.870123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519060530350859:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(329015202ns) {Configure:104(329009837ns) {ProcessConfigured:135(56093291ns) {StartInitialHandshake:157(56090175ns) {DropHandshakes:469(4741ns) {DropIncomingHandshake:441(930ns) DropOutgoingHandshake:456(1166ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:476(12060391ns) {SwitchToState:178(4993ns)}}}}}
2025-11-11T17:20:56.878051Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519060530350815:2069] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:56.888599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519060530350884:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23341
2025-11-11T17:20:56.888654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519060530350885:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23341
2025-11-11T17:20:56.913792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519056235383493:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.925951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:56.953727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519056235383493:2048] [node 2] ICP22 created new session: [1:7571519060530350886:2048]
2025-11-11T17:20:56.971344Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350885:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.985777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519060530350878:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.985823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519060530350878:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.997687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519060530350886:2048] [node 2] ICS09 handshake done sender: [1:7571519060530350815:2069] self: [1:7571519060530350814:0] peer: [2:7571519060023768013:0] socket: 531 qp: -1
2025-11-11T17:20:56.997758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519060530350886:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:57.000659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:57.006271Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519060530350878:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:57.018312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519056235383493:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(104333737ns) {HandleHandshakeStatus:329(104330843ns) {DropIncomingHandshake:441(25318ns) DropOutgoingHandshake:456(832ns) SwitchToState:178(818ns) ProcessPendingSessionEvents:495(123267ns) {ForwardSessionEventToSession:612(71568ns) ForwardSessionEventToSession:612(5664ns) ForwardSessionEventToSession:612(3538ns) ForwardSessionEventToSession:612(3297ns) ForwardSessionEventToSession:612(5854ns) ForwardSessionEventToSession:612(3727ns) ForwardSessionEventToSession:612(5583ns) ForwardSessionEventToSession:612(2474ns) ForwardSessionEventToSession:612(6224ns) ForwardSessionEventToSession:612(5933ns)}}}
2025-11-11T17:20:57.046098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519060530350859:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:57.046263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:57.046317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519060530350859:2074] [node 6] ICP22 created new session: [1:7571519064825318186:2074]
2025-11-11T17:20:57.046376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519064825318186:2074] [node 6] ICS09 handshake done sender: [1:7571519060530350878:2116] self: [1:7571519060530350875:0] peer: [6:7571519059804413738:0] socket: 587 qp: -1
2025-11-11T17:20:57.046425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519064825318186:2074] [node 6] ICS10 traffic start
2025-11-11T17:20:57.046484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:57.054364Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:57.061700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519060530350860:2068] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:56.798126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898336:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:57.086640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519063765045765:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-11T17:20:56.814107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519063282898300:2062] [node 5] ICP35 event processing took too much time PendingConnection:278(21117100ns) {HandleHandshakeStatus:329(21114447ns) {DropIncomingHandshake:441(12458ns) DropOutgoingHandshake:456(582ns) SwitchToState:178(1044ns) ProcessPendingSessionEvents:495(20799790ns) {ForwardSessionEventToSession:612(41444ns) ForwardSessionEventToSession:612(2802ns) ForwardSessionEventToSession:612(4752341ns) ForwardSessionEventToSession:612(4926ns) ForwardSessionEventToSession:612(3414ns) ForwardSessionEventToSession:612(15933396ns) ForwardSessionEventToSession:612(25322ns) ForwardSessionEventToSession:612(10881ns) ForwardSessionEventToSession:612(3366ns) ForwardSessionEventToSession:612(2359ns)}}}
2025-11-11T17:20:56.849695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519063282898336:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.869923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519058987930953:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:56.889226Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:56.897671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519058987930952:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(39846618ns) {HandleHandshakeStatus:395(39841868ns) {LogHandshakeFail:472(5081228ns) {UpdateErrorStateLog:544(10108ns)} DropOutgoingHandshake:456(1066ns) ProcessPendingSessionEvents:495(34691253ns) {DropSessionEvent:512(15762ns) DropSessionEvent:512(4842ns) DropSessionEvent:512(6263ns) DropSessionEvent:512(4084ns) DropSessionEvent:512(4746ns) DropSessionEvent:512(3084ns) DropSessionEvent:512(4929ns) DropSessionEvent:512(4156ns) DropSessionEvent:512(34599624ns) DropSessionEvent:512(33303ns)}}}
2025-11-11T17:20:56.902208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898342:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
2025-11-11T17:20:56.906540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519058987930952:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.925933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519058987930953:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16050859ns) {IncomingHandshake:261(8206857ns) {DropIncomingHandshake:441(1284ns) PrepareNewSessionHandshake:476(3692ns) {SwitchToState:178(2223ns)} GenerateSessionVirtualId:988(1218ns) IssueIncomingHandshakeReply:193(59027ns) {SwitchToState:178(118ns)}}}
2025-11-11T17:20:56.930282Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519063282898327:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:56.930313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519063282898327:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:56.931140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519058987930953:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519063282898336:2120] held: no
2025-11-11T17:20:56.931482Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519063282898336:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:56.933728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:56.933794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519058987930952:2048] [node 1] ICP22 created new session: [4:7571519063282898343:2048]
2025-11-11T17:20:56.942181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519063282898344:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55103
2025-11-11T17:20:56.957958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519063282898343:2048] [node 1] ICS09 handshake done sender: [4:7571519063282898278:2075] self: [4:7571519063282898328:0] peer: [1:7571519060530350805:0] socket: 536 qp: -1
2025-11-11T17:20:56.958145Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519063282898327:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:56.967141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519063282898343:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:56.967198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519058987930953:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:56.985735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:56.993680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:56.993747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519058987930953:2049] [node 2] ICP22 created new session: [4:7571519063282898347:2049]
2025-11-11T17:20:56.994264Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:56.998142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519058987930954:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519063282898325:2116] held: no
2025-11-11T17:20:56.998304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519063282898322:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:56.998407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:56.998453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519063282898322:2084] [node 6] ICP22 created new session: [4:7571519063282898348:2084]
2025-11-11T17:20:56.998514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519063282898348:2084] [node 6] ICS09 handshake done sender: [4:7571519063282898327:2117] self: [4:7571519063282898326:0] peer: [6:7571519059804413736:0] socket: 537 qp: -1
2025-11-11T17:20:56.998548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519063282898348:2084] [node 6] ICS10 traffic start
2025-11-11T17:20:56.998591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:56.999226Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519063282898325:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:57.003124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519063282898347:2049] [node 2] ICS09 handshake done sender: [4:7571519063282898336:2120] self: [4:7571519063282898341:0] peer: [2:7571519060023768006:0] socket: 583 qp: -1
2025-11-11T17:20:57.009698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519063282898347:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:57.021933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:57.022530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519058987930952:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(120070994ns) {HandleHandshakeStatus:329(120067856ns) {DropIncomingHandshake:441(1296ns) DropOutgoingHandshake:456(843ns) SwitchToState:178(719ns) ProcessPendingSessionEvents:495(1113ns)}}
2025-11-11T17:20:57.036319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519058987930954:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:57.045913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519058987930953:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(75681195ns) {HandleHandshakeStatus:329(66528496ns) {DropIncomingHandshake:441(985ns) DropOutgoingHandshake:456(408ns) SwitchToState:178(535ns) ProcessPendingSessionEvents:495(1259ns)}}
2025-11-11T17:20:57.053687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:57.061700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519058987930954:2050] [node 3] ICP22 created new session: [4:7571519067577865647:2050]
2025-11-11T17:20:57.073886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519067577865647:2050] [node 3] ICS09 handshake done sender: [4:7571519063282898325:2116] self: [4:7571519063282898334:0] peer: [3:7571519059470078435:0] socket: 568 qp: -1
2025-11-11T17:20:57.081910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519067577865647:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:57.081978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:57.082190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519058987930954:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(45794101ns) {HandleHandshakeStatus:329(45787464ns) {DropIncomingHandshake:441(1357ns) DropOutgoingHandshake:456(1350ns) SwitchToState:178(1015ns) ProcessPendingSessionEvents:495(1022ns)}}
2025-11-11T17:20:57.061810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519059804413645:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:57.075161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519060023768009:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:57.061894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:57.061935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519059804413645:2048] [node 1] ICP22 created new session: [6:7571519064099381044:2048]
2025-11-11T17:20:57.069991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519064099381044:2048] [node 1] ICS09 handshake done sender: [6:7571519059804413732:2120] self: [6:7571519059804413738:0] peer: [1:7571519060530350875:0] socket: 592 qp: -1
2025-11-11T17:20:57.081731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519064099381044:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:57.082882Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:57.083002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:57.102241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519059804413645:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(45332335ns) {HandleHandshakeStatus:329(40199156ns) {DropIncomingHandshake:441(1330ns) DropOutgoingHandshake:456(5095ns) SwitchToState:178(513ns) ProcessPendingSessionEvents:495(134096ns) {ForwardSessionEventToSession:612(81130ns) ForwardSessionEventToSession:612(4097ns) ForwardSessionEventToSession:612(5905ns) ForwardSessionEventToSession:612(3450ns) ForwardSessionEventToSession:612(2704ns) ForwardSessionEventToSession:612(8166ns) ForwardSessionEventToSession:612(3185ns) ForwardSessionEventToSession:612(3455ns) ForwardSessionEventToSession:612(5134ns) ForwardSessionEventToSession:612(2243ns)}}}
2025-11-11T17:20:57.102313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519059804413645:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:57.102836Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:57.075452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519055728800628:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:57.075545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:57.075591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519055728800628:2049] [node 3] ICP22 created new session: [2:7571519064318735323:2049]
2025-11-11T17:20:57.090064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519064318735323:2049] [node 3] ICS09 handshake done sender: [2:7571519060023768009:2118] self: [2:7571519060023768008:0] peer: [3:7571519063765045762:0] socket: 598 qp: -1
2025-11-11T17:20:57.096658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519064318735323:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:57.109770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:57.122136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519055175111072:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519059470078464:2120] held: no
2025-11-11T17:20:57.098033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519062495928106:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(116230702ns) {HandleHandshakeStatus:329(116226734ns) {DropIncomingHandshake:441(1309ns) DropOutgoingHandshake:456(7295ns) SwitchToState:178(1626ns) ProcessPendingSessionEvents:495(52199755ns) {ForwardSessionEventToSession:612(23965639ns) ForwardSessionEventToSession:612(3988337ns) ForwardSessionEventToSession:612(5255ns) ForwardSessionEventToSession:612(3057ns) ForwardSessionEventToSession:612(594606ns) ForwardSessionEventToSession:612(2989ns) ForwardSessionEventToSession:612(11528960ns) ForwardSessionEventToSession:612(7602ns) ForwardSessionEventToSession:612(1859647ns) ForwardSessionEventToSession:612(10216580ns)}}}
2025-11-11T17:20:57.130621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519059470078384:2058] [node 6] ICP35 event processing took too much time PendingConnection:278(115658914ns) {HandleHandshakeStatus:329(115655013ns) {DropIncomingHandshake:441(16713ns) DropOutgoingHandshake:456(702ns) SwitchToState:178(572ns) ProcessPendingSessionEvents:495(48451655ns) {ForwardSessionEventToSession:612(27782940ns) ForwardSessionEventToSession:612(1361767ns) ForwardSessionEventToSession:612(8029ns) ForwardSessionEventToSession:612(5576885ns) ForwardSessionEventToSession:612(3616ns) ForwardSessionEventToSession:612(5287701ns) ForwardSessionEventToSession:612(8457ns) ForwardSessionEventToSession:612(3284ns) ForwardSessionEventToSession:612(4100330ns) ForwardSessionEventToSession:612(4289528ns)}}}
2025-11-11T17:20:57.137673Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519059470078464:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:57.130388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519055728800628:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(54683068ns) {HandleHandshakeStatus:329(54679220ns) {DropIncomingHandshake:441(18352ns) DropOutgoingHandshake:456(663ns) SwitchToState:178(768ns) ProcessPendingSessionEvents:495(352398ns) {ForwardSessionEventToSession:612(285550ns) ForwardSessionEventToSession:612(9596ns) ForwardSessionEventToSession:612(2631ns) ForwardSessionEventToSession:612(3128ns) ForwardSessionEventToSession:612(5448ns) ForwardSessionEventToSession:612(4910ns) ForwardSessionEventToSession:612(18828ns) ForwardSessionEventToSession:612(2697ns) ForwardSessionEventToSession:612(2747ns) ForwardSessionEventToSession:612(4244ns)}}}
2025-11-11T17:20:57.141947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519055175111072:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:57.149702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:57.150360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519055175111072:2049] [node 2] ICP22 created new session: [3:7571519063765045766:2049]
2025-11-11T17:20:57.150419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519063765045766:2049] [node 2] ICS09 handshake done sender: [3:7571519059470078464:2120] self: [3:7571519063765045762:0] peer: [2:7571519060023768008:0] socket: 600 qp: -1
2025-11-11T17:20:57.150452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519063765045766:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:57.150501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:57.222435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:57.222661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519059470078455:2048] [node 1] ICS01 socket: 559 reason# EndOfStream
2025-11-11T17:20:57.222701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519055175111071:2048] [node 1] ICP30 unregister session Session# [3:7571519059470078455:2048] VirtualId# [3:7571519059470078446:0]
2025-11-11T17:20:57.222727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519059470078455:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.222792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:57.222832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:57.221745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:57.222044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519060023768021:2048] [node 1] ICS01 socket: 545 reason# EndOfStream
2025-11-11T17:20:57.222088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519055728800627:2048] [node 1] ICP30 unregister session Session# [2:7571519060023768021:2048] VirtualId# [2:7571519060023768013:0]
2025-11-11T17:20:57.222122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519060023768021:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.222203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:57.222258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:57.225656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:57.225966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519063282898343:2048] [node 1] ICS01 socket: 536 reason# EndOfStream
2025-11-11T17:20:57.226002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519058987930952:2048] [node 1] ICP30 unregister session Session# [4:7571519063282898343:2048] VirtualId# [4:7571519063282898328:0]
2025-11-11T17:20:57.226038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519063282898343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.226109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:57.226146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:57.230459Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:57.230629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519064099381044:2048] [node 1] ICS07 socket disconnect 592 reason# EndOfStream
2025-11-11T17:20:57.230654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519064099381044:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.230702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:57.230736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519064099381044:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:57.231612Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address
2025-11-11T17:20:57.231874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519059804413645:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:57.231928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519064099381044:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:57.231960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519059804413645:2048] [node 1] ICP30 unregister session Session# [6:7571519064099381044:2048] VirtualId# [6:7571519059804413738:0]
2025-11-11T17:20:57.232004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:57.232041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519059804413645:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:57.232056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:57.229773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:57.229984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519062495928209:2048] [node 1] ICS07 socket disconnect 581 reason# EndOfStream
2025-11-11T17:20:57.230011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519062495928209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.230098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:57.230138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519062495928209:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:57.232460Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address
2025-11-11T17:20:57.232691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519062495928106:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:57.232727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519062495928209:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:57.232758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519062495928106:2048] [node 1] ICP30 unregister session Session# [5:7571519062495928209:2048] VirtualId# [5:7571519062495928204:0]
2025-11-11T17:20:57.232792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:57.232818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519062495928106:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:57.232832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:57.245949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519059804413645:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:57.241802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519062495928106:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:57.281764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:57.282030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519063765045766:2049] [node 2] ICS01 socket: 600 reason# EndOfStream
2025-11-11T17:20:57.282072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519055175111072:2049] [node 2] ICP30 unregister session Session# [3:7571519063765045766:2049] VirtualId# [3:7571519063765045762:0]
2025-11-11T17:20:57.282108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519063765045766:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.282206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:57.282246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:57.285809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:57.286031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519063282898347:2049] [node 2] ICS01 socket: 583 reason# EndOfStream
2025-11-11T17:20:57.286073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519058987930953:2049] [node 2] ICP30 unregister session Session# [4:7571519063282898347:2049] VirtualId# [4:7571519063282898341:0]
2025-11-11T17:20:57.286109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519063282898347:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.286205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:57.286249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:57.286335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:57.286538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519059804413730:2049] [node 2] ICS01 socket: 567 reason# EndOfStream
2025-11-11T17:20:57.286573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519059804413646:2049] [node 2] ICP30 unregister session Session# [6:7571519059804413730:2049] VirtualId# [6:7571519059804413726:0]
2025-11-11T17:20:57.286596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519059804413730:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.286653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:57.286686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:57.286772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:57.286964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519062495928200:2049] [node 2] ICS01 socket: 569 reason# EndOfStream
2025-11-11T17:20:57.287000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519062495928107:2049] [node 2] ICP30 unregister session Session# [5:7571519062495928200:2049] VirtualId# [5:7571519062495928188:0]
2025-11-11T17:20:57.287024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519062495928200:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.287079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:57.287111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:57.425342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:57.425650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519064099381042:2050] [node 3] ICS01 socket: 593 reason# EndOfStream
2025-11-11T17:20:57.425695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519059804413647:2050] [node 3] ICP30 unregister session Session# [6:7571519064099381042:2050] VirtualId# [6:7571519059804413739:0]
2025-11-11T17:20:57.425735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519064099381042:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.425828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:57.425872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:57.429827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:57.430037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519062495928202:2050] [node 3] ICS01 socket: 558 reason# EndOfStream
2025-11-11T17:20:57.430075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519062495928108:2050] [node 3] ICP30 unregister session Session# [5:7571519062495928202:2050] VirtualId# [5:7571519062495928189:0]
2025-11-11T17:20:57.430107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519062495928202:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.430195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:57.430238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:57.433745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:57.433990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519067577865647:2050] [node 3] ICS01 socket: 568 reason# EndOfStream
2025-11-11T17:20:57.434033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519058987930954:2050] [node 3] ICP30 unregister session Session# [4:7571519067577865647:2050] VirtualId# [4:7571519063282898334:0]
2025-11-11T17:20:57.434080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519067577865647:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.434187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:57.434227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:57.565775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:57.566040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519059804413743:2060] [node 4] ICS01 socket: 542 reason# EndOfStream
2025-11-11T17:20:57.566091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519059804413681:2060] [node 4] ICP30 unregister session Session# [6:7571519059804413743:2060] VirtualId# [6:7571519059804413736:0]
2025-11-11T17:20:57.566130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519059804413743:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.566252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:57.566301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:57.569805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:57.570064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519062495928206:2068] [node 4] ICS01 socket: 554 reason# EndOfStream
2025-11-11T17:20:57.570111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519062495928130:2068] [node 4] ICP30 unregister session Session# [5:7571519062495928206:2068] VirtualId# [5:7571519062495928197:0]
2025-11-11T17:20:57.570272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519062495928206:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.570392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:57.570440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:57.653782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:57.654058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519059804413740:2065] [node 5] ICS01 socket: 547 reason# EndOfStream
2025-11-11T17:20:57.654106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519059804413698:2065] [node 5] ICP30 unregister session Session# [6:7571519059804413740:2065] VirtualId# [6:7571519059804413723:0]
2025-11-11T17:20:57.654150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519059804413740:2065] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:57.654297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:57.654355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:57.823082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519065943399732:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:57.823131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519065943399733:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:57.843387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519067451567357:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:57.843437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519067451567358:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:57.901943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519067463451665:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:57.901988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519067463451666:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:57.961053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519066312133958:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:57.961093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519066312133959:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:57.961118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519066312133960:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:58.049897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519070491590013:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:58.049952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519070491590014:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:58.049977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519070491590015:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:58.153005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519070837432712:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:58.153121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519070837432713:2049] [node 2] ICP01 ready to work
2025-11-11T17:20:58.161657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519070837432714:2050] [node 3] ICP01 ready to work
2025-11-11T17:20:58.370725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519071746534694:2087] [node 5] ICP01 ready to work
2025-11-11T17:20:58.374882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519070837432725:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.373571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519070491590028:2061] [node 4] ICP01 ready to work
2025-11-11T17:20:58.468035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519070837432731:2063] [node 5] ICP01 ready to work
2025-11-11T17:20:58.487319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519071758419015:2093] [node 5] ICP01 ready to work
2025-11-11T17:20:58.494233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519071758419016:2089] [node 6] ICP01 ready to work
2025-11-11T17:20:58.413928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519070238367091:2100] [node 4] ICP01 ready to work
2025-11-11T17:20:58.389354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590032:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:58.413959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519070238367092:2102] [node 5] ICP01 ready to work
2025-11-11T17:20:58.546506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519070837432725:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(171296029ns) {Configure:104(171290622ns) {ProcessConfigured:135(101873ns) {StartInitialHandshake:157(98589ns) {DropHandshakes:469(4757ns) {DropIncomingHandshake:441(1178ns) DropOutgoingHandshake:456(741ns)} GenerateSessionVirtualId:988(1848ns) PrepareNewSessionHandshake:476(2667ns) {SwitchToState:178(1138ns)}}}}}
2025-11-11T17:20:58.530783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519071746534719:2085] [node 4] ICP01 ready to work
2025-11-11T17:20:58.537714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519070607101303:2068] [node 5] ICP01 ready to work
2025-11-11T17:20:58.497484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519070238367094:2103] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.515154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519070238367091:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26282362ns) {Configure:104(26277179ns) {ProcessConfigured:135(91595ns) {StartInitialHandshake:157(88869ns) {DropHandshakes:469(3941ns) {DropIncomingHandshake:441(836ns) DropOutgoingHandshake:456(833ns)} GenerateSessionVirtualId:988(1751ns) PrepareNewSessionHandshake:476(2286ns) {SwitchToState:178(853ns)}}}}}
2025-11-11T17:20:58.572708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590044:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:58.538583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519071746534720:2093] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.603489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519070607101303:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40480685ns) {Configure:104(40471629ns) {ProcessConfigured:135(84170ns) {StartInitialHandshake:157(80809ns) {DropHandshakes:469(3391ns) {DropIncomingHandshake:441(832ns) DropOutgoingHandshake:456(606ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:476(1968ns) {SwitchToState:178(719ns)}}}}}
2025-11-11T17:20:58.604654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519070607101310:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:58.604852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519070607101311:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:58.559411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519067463451666:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(181277863ns) {Configure:104(181272753ns) {ProcessConfigured:135(48532ns) {StartInitialHandshake:157(46950ns) {DropHandshakes:469(2342ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(780ns)} GenerateSessionVirtualId:988(960ns) PrepareNewSessionHandshake:476(1250ns) {SwitchToState:178(499ns)}}}}}
2025-11-11T17:20:58.569970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519071758419028:2092] [node 4] ICP01 ready to work
2025-11-11T17:20:58.595893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519071758419015:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22343368ns) {Configure:104(22334393ns) {ProcessConfigured:135(20887248ns) {StartInitialHandshake:157(20883747ns) {DropHandshakes:469(4151ns) {DropIncomingHandshake:441(1378ns) DropOutgoingHandshake:456(851ns)} GenerateSessionVirtualId:988(1192ns) PrepareNewSessionHandshake:476(5243ns) {SwitchToState:178(2570ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.594621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519070837432712:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(64600463ns) {Configure:104(64594070ns) {ProcessConfigured:135(97251ns) {StartInitialHandshake:157(95103ns) {DropHandshakes:469(3489ns) {DropIncomingHandshake:441(678ns) DropOutgoingHandshake:456(810ns)} GenerateSessionVirtualId:988(1275ns) PrepareNewSessionHandshake:476(3403ns) {SwitchToState:178(1738ns)}}}}}
2025-11-11T17:20:58.646361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519067463451665:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(157451323ns) {Configure:104(157445591ns) {ProcessConfigured:135(105632ns) {StartInitialHandshake:157(103496ns) {DropHandshakes:469(724ns) {DropIncomingHandshake:441(167ns) DropOutgoingHandshake:456(256ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(2495ns) {SwitchToState:178(814ns)}}}}}
2025-11-11T17:20:58.618114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519066312133959:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(125302832ns) {Configure:104(125297179ns) {ProcessConfigured:135(93157ns) {StartInitialHandshake:157(90237ns) {DropHandshakes:469(3250ns) {DropIncomingHandshake:441(797ns) DropOutgoingHandshake:456(614ns)} GenerateSessionVirtualId:988(1023ns) PrepareNewSessionHandshake:476(3062ns) {SwitchToState:178(648ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.597223Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519070491590044:2077] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.605423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519070837432713:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(174663542ns) {Configure:104(174657373ns) {ProcessConfigured:135(20604368ns) {StartInitialHandshake:157(20600776ns) {DropHandshakes:469(3653ns) {DropIncomingHandshake:441(877ns) DropOutgoingHandshake:456(756ns)} GenerateSessionVirtualId:988(1108ns) PrepareNewSessionHandshake:476(5164ns) {SwitchToState:178(2868ns)}}}}}
2025-11-11T17:20:58.639680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519070238367099:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63768
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.625199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590045:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:58.626344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519070607101315:2067] [node 6] ICP01 ready to work
2025-11-11T17:20:58.638997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519070837432714:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(141394328ns) {Configure:104(141388795ns) {ProcessConfigured:135(95087ns) {StartInitialHandshake:157(92191ns) {DropHandshakes:469(3934ns) {DropIncomingHandshake:441(907ns) DropOutgoingHandshake:456(734ns)} GenerateSessionVirtualId:988(1044ns) PrepareNewSessionHandshake:476(1921ns) {SwitchToState:178(844ns)}}}}}
2025-11-11T17:20:58.682566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519071746534719:2085] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(124640271ns) {Configure:104(124629401ns) {ProcessConfigured:135(68877ns) {StartInitialHandshake:157(66271ns) {DropHandshakes:469(3170ns) {DropIncomingHandshake:441(927ns) DropOutgoingHandshake:456(626ns)} GenerateSessionVirtualId:988(895ns) PrepareNewSessionHandshake:476(2054ns) {SwitchToState:178(581ns)}}}}}
2025-11-11T17:20:58.682921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519071746534721:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43124
2025-11-11T17:20:58.658506Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519070491590032:2065] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.670283Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519070491590045:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.639757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519070238367100:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63768
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.673671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519071758419022:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63239
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.670908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519070837432731:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(202536010ns) {Configure:104(202529174ns) {ProcessConfigured:135(25005842ns) {StartInitialHandshake:157(25000772ns) {DropHandshakes:469(3442ns) {DropIncomingHandshake:441(837ns) DropOutgoingHandshake:456(655ns)} GenerateSessionVirtualId:988(1306ns) PrepareNewSessionHandshake:476(5006ns) {SwitchToState:178(2510ns)}}}}}
2025-11-11T17:20:58.654440Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519070607101310:2099] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.683073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519071746534723:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43124
2025-11-11T17:20:58.675787Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367100:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.694000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519070491590015:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(259777917ns) {Configure:104(259773015ns) {ProcessConfigured:135(6163653ns) {StartInitialHandshake:157(6160810ns) {DropHandshakes:469(1166ns) {DropIncomingHandshake:441(220ns) DropOutgoingHandshake:456(171ns)} GenerateSessionVirtualId:988(259ns) PrepareNewSessionHandshake:476(3082ns) {SwitchToState:178(850ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.661698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519066312133958:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(201618241ns) {Configure:104(201608280ns) {ProcessConfigured:135(45054948ns) {StartInitialHandshake:157(45051130ns) {DropHandshakes:469(2541ns) {DropIncomingHandshake:441(1116ns) DropOutgoingHandshake:456(403ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:476(3338ns) {SwitchToState:178(811ns)}}}}}
2025-11-11T17:20:58.697858Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534696:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.705909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519065943399732:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(243127845ns) {Configure:104(243119330ns) {ProcessConfigured:135(10028710ns) {StartInitialHandshake:157(10025922ns) {DropHandshakes:469(2132ns) {DropIncomingHandshake:441(291ns) DropOutgoingHandshake:456(627ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:476(3978ns) {SwitchToState:178(886ns)}}}}}
2025-11-11T17:20:58.706366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519070238367105:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63768
2025-11-11T17:20:58.710451Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519070238367094:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(187161426ns) {Configure:104(187149337ns) {ProcessConfigured:135(67008ns) {StartInitialHandshake:157(64931ns) {DropHandshakes:469(3349ns) {DropIncomingHandshake:441(702ns) DropOutgoingHandshake:456(707ns)} GenerateSessionVirtualId:988(995ns) PrepareNewSessionHandshake:476(1833ns) {SwitchToState:178(684ns)}}}}}
2025-11-11T17:20:58.725231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519070238367092:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(229089945ns) {Configure:104(229080728ns) {ProcessConfigured:135(57112426ns) {StartInitialHandshake:157(57109346ns) {DropHandshakes:469(3902ns) {DropIncomingHandshake:441(1117ns) DropOutgoingHandshake:456(562ns)} GenerateSessionVirtualId:988(1271ns) PrepareNewSessionHandshake:476(3963ns) {SwitchToState:178(1637ns)}}}}}
2025-11-11T17:20:58.725674Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367097:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.725708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519070238367097:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.734059Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367081:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.734087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519070238367081:2099] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.773864Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519070238367081:2099] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.694396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519070491590014:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(294556736ns) {Configure:104(294550593ns) {ProcessConfigured:135(8348390ns) {StartInitialHandshake:157(8345720ns) {DropHandshakes:469(3535ns) {DropIncomingHandshake:441(697ns) DropOutgoingHandshake:456(848ns)} GenerateSessionVirtualId:988(1439ns) PrepareNewSessionHandshake:476(4398ns) {SwitchToState:178(2255ns)}}}}}
2025-11-11T17:20:58.709675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519071758419022:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.785775Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519070238367097:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:58.793949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519070238367112:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63768
2025-11-11T17:20:58.697884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519071746534696:2088] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.694504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519070491590015:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.694685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519070491590014:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.723778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519070491590013:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(337277560ns) {Configure:104(337270131ns) {ProcessConfigured:135(25777242ns) {StartInitialHandshake:157(25773550ns) {DropHandshakes:469(2472ns) {DropIncomingHandshake:441(324ns) DropOutgoingHandshake:456(350ns)} GenerateSessionVirtualId:988(966ns) PrepareNewSessionHandshake:476(4498ns) {SwitchToState:178(1806ns)}}}}}
2025-11-11T17:20:58.724287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590057:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:58.745728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519070491590028:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(286605966ns) {Configure:104(286594647ns) {ProcessConfigured:135(27016507ns) {StartInitialHandshake:157(27014509ns) {DropHandshakes:469(3459ns) {DropIncomingHandshake:441(824ns) DropOutgoingHandshake:456(625ns)} GenerateSessionVirtualId:988(853ns) PrepareNewSessionHandshake:476(2190ns) {SwitchToState:178(603ns)}}}}}
2025-11-11T17:20:58.781819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519070491590028:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.662796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519066312133960:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(173976412ns) {Configure:104(173973095ns) {ProcessConfigured:135(76388ns) {StartInitialHandshake:157(66657ns) {DropHandshakes:469(1148ns) {DropIncomingHandshake:441(263ns) DropOutgoingHandshake:456(248ns)} GenerateSessionVirtualId:988(337ns) PrepareNewSessionHandshake:476(2360ns) {SwitchToState:178(1121ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.725402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519067463451665:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.798600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519065943399733:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:58.704107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519067451567357:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(238133908ns) {Configure:104(218162771ns) {ProcessConfigured:135(104024ns) {StartInitialHandshake:157(100822ns) {DropHandshakes:469(3057ns) {DropIncomingHandshake:441(599ns) DropOutgoingHandshake:456(632ns)} GenerateSessionVirtualId:988(1075ns) PrepareNewSessionHandshake:476(2825ns) {SwitchToState:178(906ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.725629Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519071758419025:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.834624Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519070491590032:2065] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.798670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:58.687439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519066312133958:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.704649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519071746534730:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43124
2025-11-11T17:20:58.718689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519071746534732:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43124
2025-11-11T17:20:58.725916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519071746534696:2088] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:58.727002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534721:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.758012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519071746534722:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43124
2025-11-11T17:20:58.805643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519067451567358:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(277391173ns) {Configure:104(277383412ns) {ProcessConfigured:135(67438052ns) {StartInitialHandshake:157(67435184ns) {DropHandshakes:469(944ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(189ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:476(4755ns) {SwitchToState:178(2102ns)}}}}}
2025-11-11T17:20:58.825829Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534732:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.835160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519067451567357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.837666Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534722:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.849841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519071746534694:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:58.853970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519071746534720:2093] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(291311179ns) {Configure:104(291303129ns) {ProcessConfigured:135(20281023ns) {StartInitialHandshake:157(20276719ns) {DropHandshakes:469(3411ns) {DropIncomingHandshake:441(784ns) DropOutgoingHandshake:456(706ns)} GenerateSessionVirtualId:988(763ns) PrepareNewSessionHandshake:476(5045ns) {SwitchToState:178(2418ns)}}}}}
2025-11-11T17:20:58.854771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:58.862858Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534725:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.862882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519071746534725:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.864347Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519071746534725:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:58.874648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519071746534694:2087] [node 5] ICP22 created new session: [2:7571519071746534741:2087]
2025-11-11T17:20:58.877671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519067451567358:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(19962224ns) {IncomingHandshake:261(19959730ns) {DropIncomingHandshake:441(1076ns) PrepareNewSessionHandshake:476(4502ns) {SwitchToState:178(3123ns)} GenerateSessionVirtualId:988(1507ns) IssueIncomingHandshakeReply:193(6489ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.801821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519070607101309:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.801874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519070607101309:2098] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.814023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519070607101322:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:58.814277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519070607101321:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:58.814697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519070607101310:2099] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:58.814880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519066312133958:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:58.814942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:58.814987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519066312133958:2048] [node 1] ICP22 created new session: [4:7571519070607101323:2048]
2025-11-11T17:20:58.815029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519070607101323:2048] [node 1] ICS09 handshake done sender: [4:7571519070607101310:2099] self: [4:7571519070607101319:0] peer: [1:7571519070238367096:0] socket: 552 qp: -1
2025-11-11T17:20:58.815060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519070607101323:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:58.815094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:58.818668Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519070607101309:2098] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:58.818886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519070607101303:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:58.818931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:58.819002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519070607101303:2068] [node 5] ICP22 created new session: [4:7571519070607101327:2068]
2025-11-11T17:20:58.819050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519070607101327:2068] [node 5] ICS09 handshake done sender: [4:7571519070607101309:2098] self: [4:7571519070607101308:0] peer: [5:7571519070491590060:0] socket: 555 qp: -1
2025-11-11T17:20:58.819084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519070607101327:2068] [node 5] ICS10 traffic start
2025-11-11T17:20:58.819121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:58.861877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519070607101320:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:58.862408Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519070607101320:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.862570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519066312133959:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.864103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519070607101330:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:58.877690Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519070607101320:2103] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:58.725671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519071758419025:2106] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.870900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519070491590014:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:58.804225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519065943399733:2049] [node 3] ICP22 created new session: [1:7571519070238367113:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.765815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519071758419025:2106] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:58.782600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519071758419037:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63239
2025-11-11T17:20:58.796559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519071758419015:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:20:58.804368Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519071758419022:2105] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:58.805633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:58.882052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519071746534719:2085] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:58.828783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519071758419039:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63239
2025-11-11T17:20:58.833671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519071758419015:2093] [node 5] ICP22 created new session: [3:7571519071758419038:2093]
2025-11-11T17:20:58.837755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519067463451665:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:58.850584Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519071758419039:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.856000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:58.857663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519071758419038:2093] [node 5] ICS09 handshake done sender: [3:7571519071758419025:2106] self: [3:7571519071758419023:0] peer: [5:7571519070491590051:0] socket: 547 qp: -1
2025-11-11T17:20:58.859088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519071758419038:2093] [node 5] ICS10 traffic start
2025-11-11T17:20:58.859149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:58.861680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519067463451665:2048] [node 1] ICP22 created new session: [3:7571519071758419040:2048]
2025-11-11T17:20:58.870734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519071758419044:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63239
2025-11-11T17:20:58.878032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519071758419040:2048] [node 1] ICS09 handshake done sender: [3:7571519071758419022:2105] self: [3:7571519071758419035:0] peer: [1:7571519070238367080:0] socket: 544 qp: -1
2025-11-11T17:20:58.885709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519071758419040:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:58.885783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:58.886226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519067463451665:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(55818270ns) {HandleHandshakeStatus:329(48291844ns) {DropIncomingHandshake:441(1135ns) DropOutgoingHandshake:456(8326ns) SwitchToState:178(438ns) ProcessPendingSessionEvents:495(110222ns) {ForwardSessionEventToSession:612(62492ns) ForwardSessionEventToSession:612(3088ns) ForwardSessionEventToSession:612(3249ns) ForwardSessionEventToSession:612(5716ns) ForwardSessionEventToSession:612(3575ns) ForwardSessionEventToSession:612(2765ns) ForwardSessionEventToSession:612(9738ns) ForwardSessionEventToSession:612(3290ns) ForwardSessionEventToSession:612(4840ns) ForwardSessionEventToSession:612(2421ns)}}}
2025-11-11T17:20:58.920308Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519071758419044:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.925828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519067463451666:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.902866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519066312133959:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:58.882138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:58.870983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:58.829062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519070238367091:2100] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.907958Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519070607101321:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.893755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519071746534719:2085] [node 4] ICP22 created new session: [2:7571519071746534743:2085]
2025-11-11T17:20:58.893825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519071746534743:2085] [node 4] ICS09 handshake done sender: [2:7571519071746534725:2113] self: [2:7571519071746534724:0] peer: [4:7571519070607101329:0] socket: 563 qp: -1
2025-11-11T17:20:58.893872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519071746534743:2085] [node 4] ICS10 traffic start
2025-11-11T17:20:58.893941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:58.871035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519070491590014:2049] [node 2] ICP22 created new session: [5:7571519070491590063:2049]
2025-11-11T17:20:58.903154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519071746534741:2087] [node 5] ICS09 handshake done sender: [2:7571519071746534696:2088] self: [2:7571519071746534695:0] peer: [5:7571519070491590052:0] socket: 541 qp: -1
2025-11-11T17:20:58.829132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.910296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519071746534741:2087] [node 5] ICS10 traffic start
2025-11-11T17:20:58.919366Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:20:58.871090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519070491590063:2049] [node 2] ICS09 handshake done sender: [5:7571519070491590032:2065] self: [5:7571519070491590052:0] peer: [2:7571519071746534695:0] socket: 542 qp: -1
2025-11-11T17:20:58.919794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519071746534747:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43124
2025-11-11T17:20:58.920445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:58.925234Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:58.942477Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534734:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.942505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519071746534734:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.945709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519067451567357:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519071746534732:2116] held: no
2025-11-11T17:20:58.949877Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519071746534732:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:58.954283Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519071746534734:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:58.946330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519071758419047:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63239
2025-11-11T17:20:58.908107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519070607101303:2068] [node 5] ICP18 handshake (actor [4:7571519070607101321:2104]) is too old
2025-11-11T17:20:58.871128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519070491590063:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:58.829175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519070238367091:2100] [node 4] ICP22 created new session: [1:7571519070238367115:2100]
2025-11-11T17:20:58.977903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519067451567357:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.946417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519071758419048:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63239
2025-11-11T17:20:58.920263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:58.921411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519070607101315:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(208070230ns) {Configure:104(208059806ns) {ProcessConfigured:135(3190982ns) {StartInitialHandshake:157(3187159ns) {DropHandshakes:469(3361ns) {DropIncomingHandshake:441(720ns) DropOutgoingHandshake:456(772ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:476(4006ns) {SwitchToState:178(1693ns)}}}}}
2025-11-11T17:20:58.941607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519066312133959:2049] [node 2] ICP22 created new session: [4:7571519070607101335:2049]
2025-11-11T17:20:58.943696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519070607101335:2049] [node 2] ICS09 handshake done sender: [4:7571519070607101320:2103] self: [4:7571519070607101329:0] peer: [2:7571519071746534724:0] socket: 564 qp: -1
2025-11-11T17:20:58.954212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519070607101335:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:58.978338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:58.981674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519070607101334:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:58.871166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:58.829233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519070238367115:2100] [node 4] ICS09 handshake done sender: [1:7571519070238367097:2110] self: [1:7571519070238367096:0] peer: [4:7571519070607101319:0] socket: 545 qp: -1
2025-11-11T17:20:59.030855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519066312133959:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(114922916ns) {HandleHandshakeStatus:329(114919124ns) {DropIncomingHandshake:441(1603ns) DropOutgoingHandshake:456(5498ns) SwitchToState:178(1047ns) ProcessPendingSessionEvents:495(19097927ns) {ForwardSessionEventToSession:612(19033739ns) ForwardSessionEventToSession:612(9830ns) ForwardSessionEventToSession:612(10888ns) ForwardSessionEventToSession:612(3239ns) ForwardSessionEventToSession:612(3286ns) ForwardSessionEventToSession:612(8511ns) ForwardSessionEventToSession:612(6602ns)}}}
2025-11-11T17:20:58.977989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:58.981828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519067451567357:2048] [node 1] ICP22 created new session: [2:7571519071746534749:2048]
2025-11-11T17:20:58.985872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519071746534749:2048] [node 1] ICS09 handshake done sender: [2:7571519071746534732:2116] self: [2:7571519071746534737:0] peer: [1:7571519070238367103:0] socket: 570 qp: -1
2025-11-11T17:20:58.985981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519067451567358:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:20:58.986091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:58.986142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519067451567358:2049] [node 3] ICP22 created new session: [2:7571519071746534750:2049]
2025-11-11T17:20:58.993822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519071746534749:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:58.993847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519071746534750:2049] [node 3] ICS09 handshake done sender: [2:7571519071746534734:2117] self: [2:7571519071746534727:0] peer: [3:7571519071758419046:0] socket: 586 qp: -1
2025-11-11T17:20:58.993891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519071746534750:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:58.993895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:58.993933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:59.001650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519067451567357:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16133713ns) {HandleHandshakeStatus:329(16130140ns) {DropIncomingHandshake:441(1355ns) DropOutgoingHandshake:456(557ns) SwitchToState:178(603ns) ProcessPendingSessionEvents:495(1138ns)}}
2025-11-11T17:20:59.032270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519071746534694:2087] [node 5] ICP35 event processing took too much time PendingConnection:278(124018665ns) {HandleHandshakeStatus:329(124014403ns) {DropIncomingHandshake:441(622ns) DropOutgoingHandshake:456(880ns) SwitchToState:178(637ns) ProcessPendingSessionEvents:495(19726600ns) {ForwardSessionEventToSession:612(19658568ns) ForwardSessionEventToSession:612(11504ns) ForwardSessionEventToSession:612(3648ns) ForwardSessionEventToSession:612(2631ns) ForwardSessionEventToSession:612(5461ns) ForwardSessionEventToSession:612(3154ns) ForwardSessionEventToSession:612(6295ns) ForwardSessionEventToSession:612(3050ns) ForwardSessionEventToSession:612(10133ns) ForwardSessionEventToSession:612(4893ns)}}}
2025-11-11T17:20:58.964454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519071758419015:2093] [node 5] ICP35 event processing took too much time PendingConnection:278(123592918ns) {HandleHandshakeStatus:329(123589830ns) {DropIncomingHandshake:441(952ns) DropOutgoingHandshake:456(1094ns) SwitchToState:178(662ns) ProcessPendingSessionEvents:495(14402232ns) {ForwardSessionEventToSession:612(14344837ns) ForwardSessionEventToSession:612(8751ns) ForwardSessionEventToSession:612(3504ns) ForwardSessionEventToSession:612(11109ns) ForwardSessionEventToSession:612(3408ns) ForwardSessionEventToSession:612(2873ns) ForwardSessionEventToSession:612(5234ns) ForwardSessionEventToSession:612(2853ns) ForwardSessionEventToSession:612(5173ns)}}}
2025-11-11T17:20:58.829268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519070238367115:2100] [node 4] ICS10 traffic start
2025-11-11T17:20:58.879341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590065:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:59.050361Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519070607101334:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.978439Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519071758419044:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:59.046081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519071746534694:2087] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:58.986080Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519071758419047:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.998001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519067463451666:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:58.998060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:59.001740Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:59.002073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519067463451666:2049] [node 2] ICP22 created new session: [3:7571519071758419049:2049]
2025-11-11T17:20:59.002128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519071758419049:2049] [node 2] ICS09 handshake done sender: [3:7571519071758419044:2112] self: [3:7571519071758419046:0] peer: [2:7571519071746534727:0] socket: 588 qp: -1
2025-11-11T17:20:59.002174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519071758419049:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:59.002229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:59.002546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519067463451666:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:20:59.042573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519071758419015:2093] [node 5] ICP18 handshake (actor [3:7571519071758419047:2113]) is too old
2025-11-11T17:20:58.829302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:58.888882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590062:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:59.054353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519066312133960:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.842829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519070238367113:2049] [node 3] ICS09 handshake done sender: [1:7571519070238367081:2099] self: [1:7571519070238367080:0] peer: [3:7571519071758419035:0] socket: 540 qp: -1
2025-11-11T17:20:58.909930Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519070491590065:2086] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:59.113169Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519070607101311:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.113305Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519070607101332:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.113325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519070607101332:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:58.913933Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:59.103976Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519071758419032:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.114798Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519070607101332:2107] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:59.107991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400044:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:58.842896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519070238367113:2049] [node 3] ICS10 traffic start
2025-11-11T17:20:58.914828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519070491590028:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519070491590045:2078] held: no
2025-11-11T17:20:59.108168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400045:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.150224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519070607101315:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:59.104032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519071758419032:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.842956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:59.169678Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534738:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.914937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519070491590013:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:59.108273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400046:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.169738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519071746534738:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:59.120475Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519071758419032:2108] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:59.179527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:58.915445Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519070491590045:2078] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:58.843522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519065943399733:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(47614503ns) {HandleHandshakeStatus:329(47611582ns) {DropIncomingHandshake:441(2178ns) DropOutgoingHandshake:456(1000ns) SwitchToState:178(646ns) ProcessPendingSessionEvents:495(112470ns) {ForwardSessionEventToSession:612(59995ns) ForwardSessionEventToSession:612(7607ns) ForwardSessionEventToSession:612(5484ns) ForwardSessionEventToSession:612(3047ns) ForwardSessionEventToSession:612(5432ns) ForwardSessionEventToSession:612(2602ns) ForwardSessionEventToSession:612(2444ns) ForwardSessionEventToSession:612(5782ns) ForwardSessionEventToSession:612(2524ns) ForwardSessionEventToSession:612(5698ns)}}}
2025-11-11T17:20:59.108369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400047:2076] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.168733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519071758419018:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.243062Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519071746534738:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:59.237736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519070607101315:2067] [node 6] ICP22 created new session: [4:7571519074902068648:2067]
2025-11-11T17:20:59.112112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519075132400044:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.849926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367105:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.929862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519070491590028:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:20:58.882281Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367104:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.882322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519070238367104:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.888382Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367112:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.908474Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519070238367104:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:58.908706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519065943399732:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:20:58.908774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:58.908822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519065943399732:2048] [node 2] ICP22 created new session: [1:7571519070238367121:2048]
2025-11-11T17:20:58.908871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519070238367121:2048] [node 2] ICS09 handshake done sender: [1:7571519070238367104:2113] self: [1:7571519070238367103:0] peer: [2:7571519071746534737:0] socket: 569 qp: -1
2025-11-11T17:20:58.908904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519070238367121:2048] [node 2] ICS10 traffic start
2025-11-11T17:20:58.908944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:58.920060Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367107:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:58.920092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519070238367107:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:20:58.989761Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519070238367107:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:59.271591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519071746534720:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:59.022037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519070238367092:2102] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:59.168777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519071758419018:2104] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:59.237795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519074902068648:2067] [node 6] ICS09 handshake done sender: [4:7571519070607101332:2107] self: [4:7571519070607101331:0] peer: [6:7571519075132400049:0] socket: 593 qp: -1
2025-11-11T17:20:58.929913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:59.112209Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519075132400046:2075] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:59.112275Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519075132400047:2076] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.280729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:59.169829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519071758419028:2092] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:20:59.026070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:59.237836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519074902068648:2067] [node 6] ICS10 traffic start
2025-11-11T17:20:59.179688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:59.285927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519071746534720:2093] [node 6] ICP22 created new session: [2:7571519076041502051:2093]
2025-11-11T17:20:59.112651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519070837432714:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:58.929968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519070491590028:2061] [node 4] ICP22 created new session: [5:7571519070491590068:2061]
2025-11-11T17:20:59.032286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519070238367092:2102] [node 5] ICP22 created new session: [1:7571519074533334421:2102]
2025-11-11T17:20:59.190720Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519071758419018:2104] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:59.237937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:59.112734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519070837432725:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:59.112777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519070837432713:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:59.146348Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519075132400045:2074] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.149752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400051:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.232810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519070837432712:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:59.266081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519070837432712:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(86314813ns) {IncomingHandshake:261(64325554ns) {DropIncomingHandshake:441(509ns) PrepareNewSessionHandshake:476(18587804ns) {SwitchToState:178(4153ns)} GenerateSessionVirtualId:988(729ns) IssueIncomingHandshakeReply:193(72943ns) {SwitchToState:178(712ns)}}}
2025-11-11T17:20:59.277819Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:59.282894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519070837432725:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519075132400046:2075] held: no
2025-11-11T17:20:59.283049Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519075132400046:2075] [node 4] ICH04 handshake succeeded
2025-11-11T17:20:59.302391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519070837432725:2059] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:59.302480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:20:59.238465Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519070607101315:2067] [node 6] ICP35 event processing took too much time PendingConnection:278(88036682ns) {HandleHandshakeStatus:329(88033298ns) {DropIncomingHandshake:441(26229ns) DropOutgoingHandshake:456(590ns) SwitchToState:178(951ns) ProcessPendingSessionEvents:495(115148ns) {ForwardSessionEventToSession:612(60822ns) ForwardSessionEventToSession:612(6414ns) ForwardSessionEventToSession:612(2587ns) ForwardSessionEventToSession:612(8251ns) ForwardSessionEventToSession:612(3145ns) ForwardSessionEventToSession:612(2352ns) ForwardSessionEventToSession:612(8457ns) ForwardSessionEventToSession:612(2885ns) ForwardSessionEventToSession:612(2555ns) ForwardSessionEventToSession:612(8635ns)}}}
2025-11-11T17:20:59.262305Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:59.262505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519074902068643:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45851
2025-11-11T17:20:59.281915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519066312133960:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519070607101334:2108] held: no
2025-11-11T17:20:59.282242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519066312133960:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(17997832ns) {HandleHandshakeStatus:395(15032681ns) {LogHandshakeFail:472(24775ns) {UpdateErrorStateLog:544(13543ns)} DropOutgoingHandshake:456(2077ns) ProcessPendingSessionEvents:495(75930ns) {DropSessionEvent:512(22059ns) DropSessionEvent:512(5472ns) DropSessionEvent:512(5933ns) DropSessionEvent:512(3992ns) DropSessionEvent:512(3918ns) DropSessionEvent:512(7476ns) DropSessionEvent:512(5417ns) DropSessionEvent:512(3660ns) DropSessionEvent:512(6071ns) DropSessionEvent:512(4236ns)}}}
2025-11-11T17:20:59.287001Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519070607101334:2108] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:59.287201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519066312133960:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:59.287293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:59.287343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519066312133960:2050] [node 3] ICP22 created new session: [4:7571519074902068654:2050]
2025-11-11T17:20:59.287390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519074902068654:2050] [node 3] ICS09 handshake done sender: [4:7571519070607101334:2108] self: [4:7571519074902068633:0] peer: [3:7571519071758419031:0] socket: 594 qp: -1
2025-11-11T17:20:59.287419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519074902068654:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:59.287490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:58.930017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519070491590068:2061] [node 4] ICS09 handshake done sender: [5:7571519070491590045:2078] self: [5:7571519070491590060:0] peer: [4:7571519070607101308:0] socket: 558 qp: -1
2025-11-11T17:20:58.930058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519070491590068:2061] [node 4] ICS10 traffic start
2025-11-11T17:20:58.930093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:58.949631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590061:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:58.978146Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519070491590044:2077] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:58.987030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519070491590015:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:58.995012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519070491590070:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:58.995265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:59.013690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519070491590015:2050] [node 3] ICP22 created new session: [5:7571519070491590071:2050]
2025-11-11T17:20:59.014415Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519070491590065:2086] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:59.021958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519070491590071:2050] [node 3] ICS09 handshake done sender: [5:7571519070491590044:2077] self: [5:7571519070491590051:0] peer: [3:7571519071758419023:0] socket: 549 qp: -1
2025-11-11T17:20:59.025241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519070491590013:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:59.025311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:59.025363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519070491590013:2048] [node 1] ICP22 created new session: [5:7571519074786557368:2048]
2025-11-11T17:20:59.025425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519074786557368:2048] [node 1] ICS09 handshake done sender: [5:7571519070491590065:2086] self: [5:7571519070491590066:0] peer: [1:7571519070238367101:0] socket: 591 qp: -1
2025-11-11T17:20:59.025474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519074786557368:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:59.025515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:59.026006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519070491590071:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:59.045898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:59.101962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519070491590015:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(114784412ns) {HandleHandshakeStatus:329(114781708ns) {DropIncomingHandshake:441(534ns) DropOutgoingHandshake:456(12081ns) SwitchToState:178(428ns) ProcessPendingSessionEvents:495(16028173ns) {ForwardSessionEventToSession:612(16022622ns)}}}
2025-11-11T17:20:59.266865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519070491590013:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(86294158ns) {HandleHandshakeStatus:329(86291049ns) {DropIncomingHandshake:441(1720ns) DropOutgoingHandshake:456(14948ns) SwitchToState:178(633ns) ProcessPendingSessionEvents:495(61710082ns) {ForwardSessionEventToSession:612(15406669ns) ForwardSessionEventToSession:612(15805294ns) ForwardSessionEventToSession:612(4591ns) ForwardSessionEventToSession:612(8137336ns) ForwardSessionEventToSession:612(9872458ns) ForwardSessionEventToSession:612(4436ns) ForwardSessionEventToSession:612(9937648ns) ForwardSessionEventToSession:612(4909ns) ForwardSessionEventToSession:612(2519ns) ForwardSessionEventToSession:612(2498176ns)}}}
2025-11-11T17:20:59.285660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519074786557405:2093] [node 6] ICP01 ready to work
2025-11-11T17:20:59.335971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519074786557406:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-11T17:20:59.379256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519074786557405:2093] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(67886523ns) {Configure:104(67878134ns) {ProcessConfigured:135(89681ns) {StartInitialHandshake:157(86979ns) {DropHandshakes:469(3782ns) {DropIncomingHandshake:441(972ns) DropOutgoingHandshake:456(718ns)} GenerateSessionVirtualId:988(1466ns) PrepareNewSessionHandshake:476(2626ns) {SwitchToState:178(787ns)}}}}}
2025-11-11T17:20:59.382077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519074786557406:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.054628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519074533334421:2102] [node 5] ICS09 handshake done sender: [1:7571519070238367107:2115] self: [1:7571519070238367101:0] peer: [5:7571519070491590066:0] socket: 587 qp: -1
2025-11-11T17:20:59.285976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519076041502051:2093] [node 6] ICS09 handshake done sender: [2:7571519071746534738:2118] self: [2:7571519071746534736:0] peer: [6:7571519075132400050:0] socket: 595 qp: -1
2025-11-11T17:20:59.196620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519071758419016:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:59.302528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519070837432725:2059] [node 4] ICP22 created new session: [6:7571519075132400095:2059]
2025-11-11T17:20:59.302578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519075132400095:2059] [node 4] ICS09 handshake done sender: [6:7571519075132400046:2075] self: [6:7571519075132400049:0] peer: [4:7571519070607101331:0] socket: 565 qp: -1
2025-11-11T17:20:59.302618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519075132400095:2059] [node 4] ICS10 traffic start
2025-11-11T17:20:59.302685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:59.322240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400099:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.322392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400100:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.322823Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519075132400045:2074] [node 1] ICH04 handshake succeeded
2025-11-11T17:20:59.323024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519070837432712:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:20:59.323130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:20:59.323179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519070837432712:2048] [node 1] ICP22 created new session: [6:7571519075132400101:2048]
2025-11-11T17:20:59.323226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519075132400101:2048] [node 1] ICS09 handshake done sender: [6:7571519075132400045:2074] self: [6:7571519075132400078:0] peer: [1:7571519070238367108:0] socket: 557 qp: -1
2025-11-11T17:20:59.323263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519075132400101:2048] [node 1] ICS10 traffic start
2025-11-11T17:20:59.323298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:20:59.324872Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519075132400047:2076] [node 2] ICH04 handshake succeeded
2025-11-11T17:20:59.325097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519070837432713:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:20:59.325189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:20:59.325260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519070837432713:2049] [node 2] ICP22 created new session: [6:7571519075132400104:2049]
2025-11-11T17:20:59.325313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519075132400104:2049] [node 2] ICS09 handshake done sender: [6:7571519075132400047:2076] self: [6:7571519075132400050:0] peer: [2:7571519071746534736:0] socket: 567 qp: -1
2025-11-11T17:20:59.325348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519075132400104:2049] [node 2] ICS10 traffic start
2025-11-11T17:20:59.325388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:20:59.385908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400098:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.389815Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:20:59.390975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400106:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.402468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519070837432714:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519075132400044:2073] held: no
2025-11-11T17:20:59.422030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519075132400106:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.422491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519070837432731:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:20:59.426821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519074786557408:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.426862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519074786557408:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:59.059406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519074533334421:2102] [node 5] ICS10 traffic start
2025-11-11T17:20:59.059489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:59.196895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519070238367092:2102] [node 5] ICP35 event processing took too much time PendingConnection:278(94356585ns) {HandleHandshakeStatus:329(94352256ns) {DropIncomingHandshake:441(31919ns) DropOutgoingHandshake:456(644ns) SwitchToState:178(784ns) ProcessPendingSessionEvents:495(24299379ns) {ForwardSessionEventToSession:612(24241034ns) ForwardSessionEventToSession:612(6026ns) ForwardSessionEventToSession:612(7123ns) ForwardSessionEventToSession:612(10908ns) ForwardSessionEventToSession:612(3660ns) ForwardSessionEventToSession:612(5429ns) ForwardSessionEventToSession:612(3598ns) ForwardSessionEventToSession:612(2805ns) ForwardSessionEventToSession:612(5286ns) ForwardSessionEventToSession:612(3151ns)}}}
2025-11-11T17:20:59.262349Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367109:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.262395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519070238367109:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:20:59.285795Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519070238367109:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:59.310327Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519070238367099:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.314215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519070238367094:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:59.329647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:59.336368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519070238367094:2103] [node 6] ICP22 created new session: [1:7571519074533334425:2103]
2025-11-11T17:20:59.345960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519074533334425:2103] [node 6] ICS09 handshake done sender: [1:7571519070238367109:2116] self: [1:7571519070238367108:0] peer: [6:7571519075132400078:0] socket: 583 qp: -1
2025-11-11T17:20:59.349929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519074533334425:2103] [node 6] ICS10 traffic start
2025-11-11T17:20:59.365754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:59.414077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519070238367094:2103] [node 6] ICP35 event processing took too much time PendingConnection:278(99536457ns) {HandleHandshakeStatus:329(99533857ns) {DropIncomingHandshake:441(819ns) DropOutgoingHandshake:456(743ns) SwitchToState:178(982ns) ProcessPendingSessionEvents:495(31759203ns) {ForwardSessionEventToSession:612(78140ns) ForwardSessionEventToSession:612(4595ns) ForwardSessionEventToSession:612(3570ns) ForwardSessionEventToSession:612(275373ns) ForwardSessionEventToSession:612(2964ns) ForwardSessionEventToSession:612(7925202ns) ForwardSessionEventToSession:612(3596165ns) ForwardSessionEventToSession:612(3377ns) ForwardSessionEventToSession:612(9669214ns) ForwardSessionEventToSession:612(49716ns)}}}
2025-11-11T17:20:59.414193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519070238367094:2103] [node 6] ICP18 handshake (actor [1:7571519070238367099:2111]) is too old
2025-11-11T17:20:59.432493Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519074786557408:2123] [node 6] ICH04 handshake succeeded
2025-11-11T17:20:59.286007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519076041502051:2093] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:20:59.232621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:59.233128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519071758419028:2092] [node 4] ICP22 created new session: [3:7571519076053386352:2092]
2025-11-11T17:20:59.254338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519076053386352:2092] [node 4] ICS09 handshake done sender: [3:7571519071758419032:2108] self: [3:7571519071758419031:0] peer: [4:7571519074902068633:0] socket: 592 qp: -1
2025-11-11T17:20:59.254395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519076053386352:2092] [node 4] ICS10 traffic start
2025-11-11T17:20:59.258694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519071758419016:2089] [node 6] ICP22 created new session: [3:7571519076053386356:2089]
2025-11-11T17:20:59.259275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:20:59.281770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519076053386356:2089] [node 6] ICS09 handshake done sender: [3:7571519071758419018:2104] self: [3:7571519071758419017:0] peer: [6:7571519075132400048:0] socket: 543 qp: -1
2025-11-11T17:20:59.284955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519076053386356:2089] [node 6] ICS10 traffic start
2025-11-11T17:20:59.285024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:59.342888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519071758419016:2089] [node 6] ICP35 event processing took too much time PendingConnection:278(146041032ns) {HandleHandshakeStatus:329(146037376ns) {DropIncomingHandshake:441(481ns) DropOutgoingHandshake:456(1026ns) SwitchToState:178(1407ns) ProcessPendingSessionEvents:495(24408972ns) {ForwardSessionEventToSession:612(24337768ns) ForwardSessionEventToSession:612(8816ns) ForwardSessionEventToSession:612(3305ns) ForwardSessionEventToSession:612(12764ns) ForwardSessionEventToSession:612(3217ns) ForwardSessionEventToSession:612(5523ns) ForwardSessionEventToSession:612(2821ns) ForwardSessionEventToSession:612(15995ns) ForwardSessionEventToSession:612(4516ns)}}}
2025-11-11T17:20:59.343551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519076053386360:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63239
2025-11-11T17:20:59.362608Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519076053386360:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.369963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519071758419016:2089] [node 6] ICP18 handshake (actor [3:7571519076053386360:2122]) is too old
2025-11-11T17:20:59.374026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519071758419028:2092] [node 4] ICP35 event processing took too much time PendingConnection:278(156241461ns) {HandleHandshakeStatus:329(156237796ns) {DropIncomingHandshake:441(16465ns) DropOutgoingHandshake:456(609ns) SwitchToState:178(833ns) ProcessPendingSessionEvents:495(47453420ns) {ForwardSessionEventToSession:612(25248668ns) ForwardSessionEventToSession:612(7781ns) ForwardSessionEventToSession:612(7596ns) ForwardSessionEventToSession:612(16330ns) ForwardSessionEventToSession:612(2566ns) ForwardSessionEventToSession:612(5982082ns) ForwardSessionEventToSession:612(4453ns) ForwardSessionEventToSession:612(8206254ns) ForwardSessionEventToSession:612(7949381ns) ForwardSessionEventToSession:612(3099ns)}}}
2025-11-11T17:20:59.286064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:59.326311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519071746534723:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:20:59.336522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519071746534720:2093] [node 6] ICP18 handshake (actor [2:7571519071746534723:2112]) is too old
2025-11-11T17:20:59.442148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519075132400108:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:20:59.443317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519070837432714:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(31452249ns) {HandleHandshakeStatus:395(31448005ns) {LogHandshakeFail:472(17525ns) {UpdateErrorStateLog:544(6127ns)} DropOutgoingHandshake:456(2174ns) ProcessPendingSessionEvents:495(19279231ns) {DropSessionEvent:512(12355ns) DropSessionEvent:512(7019ns) DropSessionEvent:512(7213196ns) DropSessionEvent:512(18828ns) DropSessionEvent:512(5496ns) DropSessionEvent:512(4700ns) DropSessionEvent:512(16037ns) DropSessionEvent:512(5493ns) DropSessionEvent:512(4596ns) DropSessionEvent:512(4891ns)}}}
2025-11-11T17:20:59.443404Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519075132400044:2073] [node 3] ICH04 handshake succeeded
2025-11-11T17:20:59.449838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519070837432714:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:20:59.461839Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519075132400106:2123] [node 5] ICH04 handshake succeeded
2025-11-11T17:20:59.462161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:20:59.462224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519070837432714:2050] [node 3] ICP22 created new session: [6:7571519075132400109:2050]
2025-11-11T17:20:59.462274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519075132400109:2050] [node 3] ICS09 handshake done sender: [6:7571519075132400044:2073] self: [6:7571519075132400048:0] peer: [3:7571519071758419017:0] socket: 548 qp: -1
2025-11-11T17:20:59.462312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519075132400109:2050] [node 3] ICS10 traffic start
2025-11-11T17:20:59.462380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:20:59.462612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519070837432714:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18912064ns) {HandleHandshakeStatus:329(18909299ns) {DropIncomingHandshake:441(4329ns) DropOutgoingHandshake:456(751ns) SwitchToState:178(894ns) ProcessPendingSessionEvents:495(1174ns)}}
2025-11-11T17:20:59.463304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519070837432731:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:20:59.463386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:20:59.463433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519070837432731:2063] [node 5] ICP22 created new session: [6:7571519075132400111:2063]
2025-11-11T17:20:59.463505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519075132400111:2063] [node 5] ICS09 handshake done sender: [6:7571519075132400106:2123] self: [6:7571519075132400107:0] peer: [5:7571519074786557407:0] socket: 562 qp: -1
2025-11-11T17:20:59.463534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519075132400111:2063] [node 5] ICS10 traffic start
2025-11-11T17:20:59.463596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:20:59.465751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519074786557405:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:20:59.465857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:20:59.465904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519074786557405:2093] [node 6] ICP22 created new session: [5:7571519074786557412:2093]
2025-11-11T17:20:59.465953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519074786557412:2093] [node 6] ICS09 handshake done sender: [5:7571519074786557408:2123] self: [5:7571519074786557407:0] peer: [6:7571519075132400107:0] socket: 556 qp: -1
2025-11-11T17:20:59.466001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519074786557412:2093] [node 6] ICS10 traffic start
2025-11-11T17:20:59.466039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:20:59.523985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:20:59.524242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519071746534749:2048] [node 1] ICS01 socket: 570 reason# EndOfStream
2025-11-11T17:20:59.524288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519067451567357:2048] [node 1] ICP30 unregister session Session# [2:7571519071746534749:2048] VirtualId# [2:7571519071746534737:0]
2025-11-11T17:20:59.524325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519071746534749:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.524428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:59.524472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:59.526039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:20:59.526247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519071758419040:2048] [node 1] ICS07 socket disconnect 544 reason# EndOfStream
2025-11-11T17:20:59.526275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519071758419040:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.526347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:59.526389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519071758419040:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:59.527147Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address
2025-11-11T17:20:59.527507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519067463451665:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.527556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519071758419040:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.527593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519067463451665:2048] [node 1] ICP30 unregister session Session# [3:7571519071758419040:2048] VirtualId# [3:7571519071758419035:0]
2025-11-11T17:20:59.527641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:59.527662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519067463451665:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.527678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.537757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:20:59.538003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519070607101323:2048] [node 1] ICS07 socket disconnect 552 reason# EndOfStream
2025-11-11T17:20:59.538036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519070607101323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.538127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:59.538168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519070607101323:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:59.538929Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address
2025-11-11T17:20:59.539222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519066312133958:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.539275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519070607101323:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.539310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519066312133958:2048] [node 1] ICP30 unregister session Session# [4:7571519070607101323:2048] VirtualId# [4:7571519070607101319:0]
2025-11-11T17:20:59.539355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:59.539387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519066312133958:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.539405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.538487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519067463451665:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:59.545777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:20:59.546024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519074786557368:2048] [node 1] ICS07 socket disconnect 591 reason# EndOfStream
2025-11-11T17:20:59.546053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519074786557368:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.546181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:59.546224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519074786557368:2048] [node 1] ICS15 start handshake
2025-11-11T17:20:59.546816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:20:59.547020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519075132400101:2048] [node 1] ICS01 socket: 557 reason# EndOfStream
2025-11-11T17:20:59.547055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519070837432712:2048] [node 1] ICP30 unregister session Session# [6:7571519075132400101:2048] VirtualId# [6:7571519075132400078:0]
2025-11-11T17:20:59.547092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519075132400101:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.547162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:20:59.547213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:59.548805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519066312133958:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:59.551569Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address
2025-11-11T17:20:59.551870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519070491590013:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.551921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519074786557368:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.551957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519070491590013:2048] [node 1] ICP30 unregister session Session# [5:7571519074786557368:2048] VirtualId# [5:7571519070491590066:0]
2025-11-11T17:20:59.552008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:20:59.552041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519070491590013:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.552058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.561733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519070491590013:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:20:59.591715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:20:59.591987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519071758419049:2049] [node 2] ICS01 socket: 588 reason# EndOfStream
2025-11-11T17:20:59.592030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519067463451666:2049] [node 2] ICP30 unregister session Session# [3:7571519071758419049:2049] VirtualId# [3:7571519071758419046:0]
2025-11-11T17:20:59.592067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519071758419049:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.592156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:59.592201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:59.597581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:20:59.597786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:20:59.597968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519075132400104:2049] [node 2] ICS07 socket disconnect 567 reason# EndOfStream
2025-11-11T17:20:59.597990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519075132400104:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.598054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:59.598105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519075132400104:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:59.597690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:20:59.597895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519070607101335:2049] [node 2] ICS07 socket disconnect 564 reason# EndOfStream
2025-11-11T17:20:59.597924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519070607101335:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.597999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:59.598041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519070607101335:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:59.597867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519070491590063:2049] [node 2] ICS07 socket disconnect 542 reason# EndOfStream
2025-11-11T17:20:59.597908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519070491590063:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.597995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:20:59.598036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519070491590063:2049] [node 2] ICS15 start handshake
2025-11-11T17:20:59.599100Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address
2025-11-11T17:20:59.599615Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address
2025-11-11T17:20:59.599911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519070837432713:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.599217Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address
2025-11-11T17:20:59.599768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519066312133959:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.599828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519070607101335:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.599957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519066312133959:2049] [node 2] ICP30 unregister session Session# [4:7571519070607101335:2049] VirtualId# [4:7571519070607101329:0]
2025-11-11T17:20:59.600117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:59.600153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519066312133959:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.600167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.599531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519070491590014:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.599698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519070491590063:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.599893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519070491590014:2049] [node 2] ICP30 unregister session Session# [5:7571519070491590063:2049] VirtualId# [5:7571519070491590052:0]
2025-11-11T17:20:59.600515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:59.600524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519075132400104:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.600864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519070837432713:2049] [node 2] ICP30 unregister session Session# [6:7571519075132400104:2049] VirtualId# [6:7571519075132400050:0]
2025-11-11T17:20:59.600907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:20:59.600944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519070837432713:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.600831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519070491590014:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.605480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.600856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.613775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519066312133959:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:59.616289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519070837432713:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:59.611177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519070491590014:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:20:59.649810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:20:59.650063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519074902068654:2050] [node 3] ICS01 socket: 594 reason# EndOfStream
2025-11-11T17:20:59.650100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519066312133960:2050] [node 3] ICP30 unregister session Session# [4:7571519074902068654:2050] VirtualId# [4:7571519074902068633:0]
2025-11-11T17:20:59.650135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519074902068654:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.650226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:59.650268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:59.649803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:20:59.650042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519070491590071:2050] [node 3] ICS07 socket disconnect 549 reason# EndOfStream
2025-11-11T17:20:59.650071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519070491590071:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.650145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:59.650183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519070491590071:2050] [node 3] ICS15 start handshake
2025-11-11T17:20:59.650551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:20:59.650734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519075132400109:2050] [node 3] ICS01 socket: 548 reason# EndOfStream
2025-11-11T17:20:59.650768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519070837432714:2050] [node 3] ICP30 unregister session Session# [6:7571519075132400109:2050] VirtualId# [6:7571519075132400048:0]
2025-11-11T17:20:59.650795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519075132400109:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.650874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:20:59.650919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:59.651004Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Couldn't connect to any resolved address
2025-11-11T17:20:59.660091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519070491590015:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.660165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519070491590071:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.660209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519070491590015:2050] [node 3] ICP30 unregister session Session# [5:7571519070491590071:2050] VirtualId# [5:7571519070491590051:0]
2025-11-11T17:20:59.660262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:20:59.660298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519070491590015:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.660313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2039) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.672807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519070491590015:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:20:59.739864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:20:59.740174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519075132400095:2059] [node 4] ICS01 socket: 565 reason# EndOfStream
2025-11-11T17:20:59.740216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519070837432725:2059] [node 4] ICP30 unregister session Session# [6:7571519075132400095:2059] VirtualId# [6:7571519075132400049:0]
2025-11-11T17:20:59.740255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519075132400095:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.740379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:59.740432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:59.741732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:20:59.741961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519070491590068:2061] [node 4] ICS07 socket disconnect 558 reason# EndOfStream
2025-11-11T17:20:59.741990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519070491590068:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.742083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:20:59.742124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519070491590068:2061] [node 4] ICS15 start handshake
2025-11-11T17:20:59.742878Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Couldn't connect to any resolved address
2025-11-11T17:20:59.743143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519070491590028:2061] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:20:59.743197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519070491590068:2061] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:20:59.743232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519070491590028:2061] [node 4] ICP30 unregister session Session# [5:7571519070491590068:2061] VirtualId# [5:7571519070491590060:0]
2025-11-11T17:20:59.743275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:20:59.743306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519070491590028:2061] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.743323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7091) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:20:59.813761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:20:59.813995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519075132400111:2063] [node 5] ICS01 socket: 562 reason# EndOfStream
2025-11-11T17:20:59.814040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519070837432731:2063] [node 5] ICP30 unregister session Session# [6:7571519075132400111:2063] VirtualId# [6:7571519075132400107:0]
2025-11-11T17:20:59.814075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519075132400111:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:20:59.814198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:20:59.814243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:20:59.945746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519074423855863:2048] [node 2] ICP01 ready to work
2025-11-11T17:20:59.945793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519074423855864:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:59.966581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519073393952028:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:59.966623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519073393952029:2049] [node 3] ICP01 ready to work
2025-11-11T17:20:59.991759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519073029112303:2048] [node 1] ICP01 ready to work
2025-11-11T17:20:59.991820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519073029112304:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:00.017856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519077360414830:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:00.017898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519077360414831:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:00.017929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519077360414832:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:00.039077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519078041656034:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:00.039154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519078041656035:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:00.039265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519078041656036:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:00.062715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519077375158610:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:00.062774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519077375158611:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:00.062811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519077375158612:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:00.606807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519077375158629:2065] [node 5] ICP01 ready to work
2025-11-11T17:21:00.778206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519078718823223:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:00.793898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519077360414893:2103] [node 6] ICP01 ready to work
2025-11-11T17:21:00.802708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519077688919387:2107] [node 6] ICP01 ready to work
2025-11-11T17:21:00.806558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519078718823224:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:00.819858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519077688919388:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:00.819877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519078718823225:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:00.851030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519077375158629:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(237177132ns) {Configure:104(237164347ns) {ProcessConfigured:135(66560ns) {StartInitialHandshake:157(64072ns) {DropHandshakes:469(1583ns) {DropIncomingHandshake:441(419ns) DropOutgoingHandshake:456(324ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:476(1957ns) {SwitchToState:178(938ns)}}}}}
2025-11-11T17:21:00.851512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519077375158674:2069] [node 4] ICP01 ready to work
2025-11-11T17:21:00.845296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519077360414894:2106] [node 5] ICP01 ready to work
2025-11-11T17:21:00.849718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519077688919389:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:00.854806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519078041656035:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(140243500ns) {Configure:104(140228323ns) {ProcessConfigured:135(4668917ns) {StartInitialHandshake:157(4665165ns) {DropHandshakes:469(2750ns) {DropIncomingHandshake:441(606ns) DropOutgoingHandshake:456(617ns)} GenerateSessionVirtualId:988(608ns) PrepareNewSessionHandshake:476(5317ns) {SwitchToState:178(2720ns)}}}}}
2025-11-11T17:21:00.882615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519077375158611:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30605903ns) {Configure:104(30598393ns) {ProcessConfigured:135(71914ns) {StartInitialHandshake:157(69228ns) {DropHandshakes:469(2096ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(950ns) PrepareNewSessionHandshake:476(2249ns) {SwitchToState:178(1020ns)}}}}}
2025-11-11T17:21:00.886190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519078041656099:2067] [node 4] ICP01 ready to work
2025-11-11T17:21:00.889716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519074423855863:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(40710924ns) {Configure:104(40706312ns) {ProcessConfigured:135(17927143ns) {StartInitialHandshake:157(17923360ns) {DropHandshakes:469(3093ns) {DropIncomingHandshake:441(572ns) DropOutgoingHandshake:456(476ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:476(5725ns) {SwitchToState:178(2145ns)}}}}}
2025-11-11T17:21:00.886594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519078041656100:2070] [node 6] ICP01 ready to work
2025-11-11T17:21:00.887378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519073029112304:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(265172538ns) {Configure:104(246192872ns) {ProcessConfigured:135(76655ns) {StartInitialHandshake:157(54009ns) {DropHandshakes:469(2071ns) {DropIncomingHandshake:441(322ns) DropOutgoingHandshake:456(615ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:476(1061ns) {SwitchToState:178(361ns)}}}}}
2025-11-11T17:21:00.903961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519077688919387:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26089379ns) {Configure:104(26085144ns) {ProcessConfigured:135(57621ns) {StartInitialHandshake:157(55771ns) {DropHandshakes:469(1387ns) {DropIncomingHandshake:441(380ns) DropOutgoingHandshake:456(292ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:476(1108ns) {SwitchToState:178(465ns)}}}}}
2025-11-11T17:21:00.912212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519077360414894:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21973275ns) {Configure:104(21966261ns) {ProcessConfigured:135(57271ns) {StartInitialHandshake:157(54214ns) {DropHandshakes:469(2939ns) {DropIncomingHandshake:441(1221ns) DropOutgoingHandshake:456(479ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:476(1513ns) {SwitchToState:178(609ns)}}}}}
2025-11-11T17:21:00.916561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519077324079664:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:00.918120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519077324079665:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:00.927625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519078041656109:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:00.918423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519077324079666:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:00.950485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519077360414905:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:00.950024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519078718823223:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(52082620ns) {Configure:104(50018657ns) {ProcessConfigured:135(1092451ns) {StartInitialHandshake:157(1087637ns) {DropHandshakes:469(3329ns) {DropIncomingHandshake:441(768ns) DropOutgoingHandshake:456(359ns)} GenerateSessionVirtualId:988(984ns) PrepareNewSessionHandshake:476(4574ns) {SwitchToState:178(2310ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:00.986052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519078718823224:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(35712824ns) {Configure:104(35708038ns) {ProcessConfigured:135(60990ns) {StartInitialHandshake:157(57291ns) {DropHandshakes:469(2655ns) {DropIncomingHandshake:441(1248ns) DropOutgoingHandshake:456(504ns)} GenerateSessionVirtualId:988(1056ns) PrepareNewSessionHandshake:476(1757ns) {SwitchToState:178(532ns)}}}}}
2025-11-11T17:21:00.974376Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519077360414905:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:00.992420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519078718823236:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60697
2025-11-11T17:21:00.978428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519077324079664:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(58597260ns) {Configure:104(58592302ns) {ProcessConfigured:135(71428ns) {StartInitialHandshake:157(68612ns) {DropHandshakes:469(2780ns) {DropIncomingHandshake:441(703ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(758ns) PrepareNewSessionHandshake:476(1671ns) {SwitchToState:178(660ns)}}}}}
2025-11-11T17:21:00.994459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519077688919388:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(76685573ns) {Configure:104(76678760ns) {ProcessConfigured:135(71327ns) {StartInitialHandshake:157(68634ns) {DropHandshakes:469(2599ns) {DropIncomingHandshake:441(457ns) DropOutgoingHandshake:456(406ns)} GenerateSessionVirtualId:988(574ns) PrepareNewSessionHandshake:476(1461ns) {SwitchToState:178(600ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:00.974702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519077360414909:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:01.000859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519077688919400:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4397
2025-11-11T17:21:01.001101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519077688919401:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4397
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.003070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519077324079666:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21407150ns) {Configure:104(21403343ns) {ProcessConfigured:135(57364ns) {StartInitialHandshake:157(54213ns) {DropHandshakes:469(2273ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(778ns) PrepareNewSessionHandshake:476(7197ns) {SwitchToState:178(906ns)}}}}}
2025-11-11T17:21:01.004296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519081619046971:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55400
2025-11-11T17:21:01.004353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519081619046972:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55400
2025-11-11T17:21:01.004403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519081619046973:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.028168Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519081619046971:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.006998Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519078041656109:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.001206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519077688919402:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4397
2025-11-11T17:21:01.001280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519077688919403:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4397
2025-11-11T17:21:01.034255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519077375158688:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.034340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670125985:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.046482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519081655382207:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:01.047277Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519077360414909:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.047862Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519077375158688:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.048004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519077375158674:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.049490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670125987:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.007610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623409:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.048355Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519077360414896:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.041919Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919401:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.062751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519082336623409:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.062993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519078041656036:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.074780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519083013790536:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60697
2025-11-11T17:21:01.074928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519083013790537:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60697
2025-11-11T17:21:01.058022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919400:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.048386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519077360414896:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.084697Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519083013790536:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.084836Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519078718823229:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.084860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519078718823229:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.090141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519077324079668:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.090180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519077324079668:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.091431Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519077324079668:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:01.065294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519081983886703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4397
2025-11-11T17:21:01.077759Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519077375158688:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:01.049766Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519077360414896:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:01.092856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519077324079664:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.092967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:01.093023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519077324079664:2107] [node 5] ICP22 created new session: [3:7571519081619046979:2107]
2025-11-11T17:21:01.093071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519081619046979:2107] [node 5] ICS09 handshake done sender: [3:7571519077324079668:2111] self: [3:7571519077324079667:0] peer: [5:7571519082336623411:0] socket: 574 qp: -1
2025-11-11T17:21:01.093101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519081619046979:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:01.093159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.071374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519077360414893:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.071496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:01.071551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519077360414893:2103] [node 6] ICP22 created new session: [4:7571519081655382210:2103]
2025-11-11T17:21:01.071619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519081655382210:2103] [node 6] ICS09 handshake done sender: [4:7571519077360414896:2110] self: [4:7571519077360414895:0] peer: [6:7571519081670125986:0] socket: 554 qp: -1
2025-11-11T17:21:01.071663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519081655382210:2103] [node 6] ICS10 traffic start
2025-11-11T17:21:01.071702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:01.098310Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519078718823229:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:01.125847Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3176) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:01.127264Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519081670125985:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.127319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519077375158674:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.127432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:01.127505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519077375158674:2069] [node 4] ICP22 created new session: [6:7571519081670125988:2069]
2025-11-11T17:21:01.127549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519081670125988:2069] [node 4] ICS09 handshake done sender: [6:7571519077375158688:2115] self: [6:7571519081670125986:0] peer: [4:7571519077360414895:0] socket: 558 qp: -1
2025-11-11T17:21:01.127584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519081670125988:2069] [node 4] ICS10 traffic start
2025-11-11T17:21:01.127621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:01.127883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519077375158674:2069] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:01.128547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519077375158611:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.073622Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919402:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.081658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919403:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.081800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519073393952028:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.117505Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519081983886703:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.117994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519081983886708:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4397
2025-11-11T17:21:01.134143Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.137052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519078041656036:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519082336623409:2116] held: no
2025-11-11T17:21:01.137377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623412:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
2025-11-11T17:21:01.135397Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919393:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.135459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519077688919393:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.136237Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519077688919403:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:01.137964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519073393952028:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.138024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:01.138070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519073393952028:2048] [node 1] ICP22 created new session: [2:7571519081983886711:2048]
2025-11-11T17:21:01.138117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519081983886711:2048] [node 1] ICS09 handshake done sender: [2:7571519077688919403:2118] self: [2:7571519081983886707:0] peer: [1:7571519078718823228:0] socket: 566 qp: -1
2025-11-11T17:21:01.138155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519081983886711:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:01.138225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.144434Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519081619046973:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.144817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519081619046972:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.145249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519073029112304:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.145310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519073029112303:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.145387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519081619046981:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.147470Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519077688919393:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:01.141669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623413:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
2025-11-11T17:21:01.143093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519082336623413:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.143289Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519082336623409:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:01.142740Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519081655382207:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.146840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519077360414830:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.147194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519081655382213:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:01.142049Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519078718823236:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.142120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519074423855863:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.142184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:01.142228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519074423855863:2048] [node 2] ICP22 created new session: [1:7571519083013790540:2048]
2025-11-11T17:21:01.142300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519083013790540:2048] [node 2] ICS09 handshake done sender: [1:7571519078718823229:2112] self: [1:7571519078718823228:0] peer: [2:7571519081983886707:0] socket: 561 qp: -1
2025-11-11T17:21:01.142342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519083013790540:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:01.142373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:01.140728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670125992:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.154587Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919395:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.154631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519077688919395:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.155395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519077688919387:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.155497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:01.155546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519077688919387:2107] [node 6] ICP22 created new session: [2:7571519081983886713:2107]
2025-11-11T17:21:01.155639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519081983886713:2107] [node 6] ICS09 handshake done sender: [2:7571519077688919393:2111] self: [2:7571519077688919392:0] peer: [6:7571519081670125991:0] socket: 568 qp: -1
2025-11-11T17:21:01.155680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519081983886713:2107] [node 6] ICS10 traffic start
2025-11-11T17:21:01.155719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:01.160071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519081670125985:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:01.161824Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:01.163397Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519081619046981:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.164254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519081619046985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.159259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519078041656036:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.160413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:01.160473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519078041656036:2050] [node 3] ICP22 created new session: [5:7571519082336623415:2050]
2025-11-11T17:21:01.160527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519082336623415:2050] [node 3] ICS09 handshake done sender: [5:7571519082336623409:2116] self: [5:7571519082336623411:0] peer: [3:7571519077324079667:0] socket: 576 qp: -1
2025-11-11T17:21:01.160569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519082336623415:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:01.160642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:01.161239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519078041656099:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.162878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519077360414902:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.162920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519077360414902:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.165337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519081655382215:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:01.165945Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519083013790537:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.166057Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519078718823227:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.166079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519078718823227:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.167811Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519078718823231:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.167841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519078718823231:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.169353Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519077360414902:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.170123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670125993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.170177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519077375158611:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.170268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:01.170308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519077375158611:2049] [node 2] ICP22 created new session: [6:7571519081670125994:2049]
2025-11-11T17:21:01.170359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519081670125994:2049] [node 2] ICS09 handshake done sender: [6:7571519081670125985:2116] self: [6:7571519081670125991:0] peer: [2:7571519077688919392:0] socket: 578 qp: -1
2025-11-11T17:21:01.170394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519081670125994:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:01.170439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:01.170161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519077688919395:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:01.169902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519081619046987:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.174150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519073393952029:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.174220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:01.174265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519073393952029:2049] [node 3] ICP22 created new session: [2:7571519081983886718:2049]
2025-11-11T17:21:01.174313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519081983886718:2049] [node 3] ICS09 handshake done sender: [2:7571519077688919395:2112] self: [2:7571519077688919394:0] peer: [3:7571519081619046983:0] socket: 569 qp: -1
2025-11-11T17:21:01.174345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519081983886718:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:01.174382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:01.171313Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519078718823227:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:01.171178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519077375158611:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.175585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623418:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
2025-11-11T17:21:01.175654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
2025-11-11T17:21:01.172434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670125996:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.176214Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519081655382213:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.176270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519077360414894:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.176355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:01.176408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519077360414894:2106] [node 5] ICP22 created new session: [4:7571519081655382218:2106]
2025-11-11T17:21:01.176446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519081655382218:2106] [node 5] ICS09 handshake done sender: [4:7571519077360414902:2113] self: [4:7571519077360414901:0] peer: [5:7571519082336623417:0] socket: 586 qp: -1
2025-11-11T17:21:01.176475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519081655382218:2106] [node 5] ICS10 traffic start
2025-11-11T17:21:01.176510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:01.177805Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519081619046973:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:01.178225Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519082336623413:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:01.178523Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519081619046972:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:01.178780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519078041656099:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.178867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:01.178913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519078041656099:2067] [node 4] ICP22 created new session: [5:7571519082336623420:2067]
2025-11-11T17:21:01.178985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519082336623420:2067] [node 4] ICS09 handshake done sender: [5:7571519082336623413:2118] self: [5:7571519082336623417:0] peer: [4:7571519077360414901:0] socket: 570 qp: -1
2025-11-11T17:21:01.179017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519082336623420:2067] [node 4] ICS10 traffic start
2025-11-11T17:21:01.179074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:01.186257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519073029112304:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.186370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:01.186425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519073029112304:2049] [node 2] ICP22 created new session: [3:7571519081619046988:2049]
2025-11-11T17:21:01.186506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519081619046988:2049] [node 2] ICS09 handshake done sender: [3:7571519081619046973:2117] self: [3:7571519081619046983:0] peer: [2:7571519077688919394:0] socket: 572 qp: -1
2025-11-11T17:21:01.186548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519081619046988:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:01.186604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:01.187194Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:01.188105Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519081670125993:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.193796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519077375158629:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.188773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519081655382217:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:01.188899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519077360414832:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.195626Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519078041656106:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.195659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519078041656106:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.193228Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519078718823231:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:01.197122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519074423855864:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.197176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:01.197227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519074423855864:2049] [node 3] ICP22 created new session: [1:7571519083013790549:2049]
2025-11-11T17:21:01.197276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519083013790549:2049] [node 3] ICS09 handshake done sender: [1:7571519078718823227:2111] self: [1:7571519078718823226:0] peer: [3:7571519081619046984:0] socket: 560 qp: -1
2025-11-11T17:21:01.197309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519083013790549:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:01.197346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:01.197850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519078718823223:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.194160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519073029112303:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.194260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:01.194310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519073029112303:2048] [node 1] ICP22 created new session: [3:7571519081619046990:2048]
2025-11-11T17:21:01.194354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519081619046990:2048] [node 1] ICS09 handshake done sender: [3:7571519081619046972:2116] self: [3:7571519081619046984:0] peer: [1:7571519078718823226:0] socket: 571 qp: -1
2025-11-11T17:21:01.194390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519081619046990:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:01.194421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:01.195190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519073029112304:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:01.201832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519077324079672:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.201858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519077324079672:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.197900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:01.197942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519078718823223:2107] [node 4] ICP22 created new session: [1:7571519083013790551:2107]
2025-11-11T17:21:01.197983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519083013790551:2107] [node 4] ICS09 handshake done sender: [1:7571519078718823231:2113] self: [1:7571519078718823230:0] peer: [4:7571519081655382214:0] socket: 562 qp: -1
2025-11-11T17:21:01.198016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519083013790551:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:01.198043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:01.201301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519083013790547:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60697
2025-11-11T17:21:01.201437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519083013790548:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60697
2025-11-11T17:21:01.212055Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519083013790547:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.203701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670125999:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.202142Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519078041656106:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:01.213268Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519081670125996:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.213355Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519081670125993:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:01.210184Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519081655382215:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.210295Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:01.214509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519077375158629:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.214606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:01.214655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519077375158629:2065] [node 5] ICP22 created new session: [6:7571519081670126000:2065]
2025-11-11T17:21:01.214711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519081670126000:2065] [node 5] ICS09 handshake done sender: [6:7571519081670125993:2119] self: [6:7571519081670125998:0] peer: [5:7571519078041656105:0] socket: 582 qp: -1
2025-11-11T17:21:01.214752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519081670126000:2065] [node 5] ICS10 traffic start
2025-11-11T17:21:01.214785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:01.215198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519077375158610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.217558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519077360414830:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519081655382207:2117] held: no
2025-11-11T17:21:01.217745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519081655382207:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.223192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519077360414830:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.223416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:01.223660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519077360414830:2048] [node 1] ICP22 created new session: [4:7571519081655382222:2048]
2025-11-11T17:21:01.223138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519081619046992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55400
2025-11-11T17:21:01.223574Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519082336623418:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.223858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519081655382222:2048] [node 1] ICS09 handshake done sender: [4:7571519081655382207:2117] self: [4:7571519081655382214:0] peer: [1:7571519078718823230:0] socket: 579 qp: -1
2025-11-11T17:21:01.223852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519078041656100:2070] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.223987Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519077324079672:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:01.223915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519081655382222:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:01.223906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:01.223952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519078041656100:2070] [node 6] ICP22 created new session: [5:7571519082336623425:2070]
2025-11-11T17:21:01.223994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519082336623425:2070] [node 6] ICS09 handshake done sender: [5:7571519078041656106:2113] self: [5:7571519078041656105:0] peer: [6:7571519081670125998:0] socket: 567 qp: -1
2025-11-11T17:21:01.224375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519082336623425:2070] [node 6] ICS10 traffic start
2025-11-11T17:21:01.224437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.241012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519077324079665:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.241104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:01.241159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519077324079665:2108] [node 4] ICP22 created new session: [3:7571519081619046995:2108]
2025-11-11T17:21:01.223962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:01.241208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519081619046995:2108] [node 4] ICS09 handshake done sender: [3:7571519077324079672:2113] self: [3:7571519077324079671:0] peer: [4:7571519081655382221:0] socket: 590 qp: -1
2025-11-11T17:21:01.226444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519077360414831:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.227124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519081655382224:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:01.227676Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519081655382213:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:01.253872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519077360414832:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.254023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:01.254080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519077360414832:2050] [node 3] ICP22 created new session: [4:7571519081655382226:2050]
2025-11-11T17:21:01.254131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519081655382226:2050] [node 3] ICS09 handshake done sender: [4:7571519081655382213:2118] self: [4:7571519081655382221:0] peer: [3:7571519077324079671:0] socket: 591 qp: -1
2025-11-11T17:21:01.254180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519081655382226:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:01.254220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.239421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670126003:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.265105Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519081670126003:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.265296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519077375158612:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.233836Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919397:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.262279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623424:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
2025-11-11T17:21:01.262492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519078041656035:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.268670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623428:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
2025-11-11T17:21:01.241247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519081619046995:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:01.233888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519077688919397:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.242029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519077688919397:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:01.242322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519077688919388:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.242386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:01.242428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519077688919388:2108] [node 4] ICP22 created new session: [2:7571519081983886721:2108]
2025-11-11T17:21:01.242473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519081983886721:2108] [node 4] ICS09 handshake done sender: [2:7571519077688919397:2113] self: [2:7571519077688919396:0] peer: [4:7571519081655382225:0] socket: 592 qp: -1
2025-11-11T17:21:01.242504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519081983886721:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:01.242539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:01.267060Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519077688919399:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.267087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519077688919399:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.254409Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519078718823233:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.241285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:01.254467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519078718823233:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.271580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519082336623418:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:01.270217Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519077688919399:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:01.241831Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519081619046992:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.243523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519077324079665:2108] [node 4] ICP18 handshake (actor [3:7571519081619046992:2121]) is too old
2025-11-11T17:21:01.269564Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519081619046970:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.269601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519081619046970:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.283573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519077360414832:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(16477675ns) {HandleHandshakeStatus:329(16465751ns) {DropIncomingHandshake:441(1245ns) DropOutgoingHandshake:456(11770ns) SwitchToState:178(492ns) ProcessPendingSessionEvents:495(15936164ns) {ForwardSessionEventToSession:612(8833538ns) ForwardSessionEventToSession:612(7515ns) ForwardSessionEventToSession:612(3786ns) ForwardSessionEventToSession:612(2552286ns) ForwardSessionEventToSession:612(4154ns) ForwardSessionEventToSession:612(4039094ns) ForwardSessionEventToSession:612(4836ns) ForwardSessionEventToSession:612(3419ns) ForwardSessionEventToSession:612(464015ns) ForwardSessionEventToSession:612(3919ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.291331Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519081619046970:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:01.291417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670126005:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.282661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519077688919389:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.292513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519078041656035:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.292644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:01.280433Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519083013790548:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.282108Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519078718823233:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:01.282399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519078718823224:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.282458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:01.287385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519078718823224:2108] [node 6] ICP22 created new session: [1:7571519083013790557:2108]
2025-11-11T17:21:01.287439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519083013790557:2108] [node 6] ICS09 handshake done sender: [1:7571519078718823233:2114] self: [1:7571519078718823232:0] peer: [6:7571519081670126002:0] socket: 597 qp: -1
2025-11-11T17:21:01.287508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519083013790557:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:01.287547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:01.282728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:01.292991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519078041656035:2049] [node 2] ICP22 created new session: [5:7571519082336623429:2049]
2025-11-11T17:21:01.282784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519077688919389:2109] [node 5] ICP22 created new session: [2:7571519081983886724:2109]
2025-11-11T17:21:01.282830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519081983886724:2109] [node 5] ICS09 handshake done sender: [2:7571519077688919399:2114] self: [2:7571519077688919398:0] peer: [5:7571519082336623427:0] socket: 593 qp: -1
2025-11-11T17:21:01.282869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519081983886724:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:01.282908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.297986Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:01.293063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519082336623429:2049] [node 2] ICS09 handshake done sender: [5:7571519082336623418:2119] self: [5:7571519082336623427:0] peer: [2:7571519077688919398:0] socket: 547 qp: -1
2025-11-11T17:21:01.293107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519082336623429:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:01.293143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:01.295731Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519082336623424:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.295862Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:01.304544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519078041656035:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:01.304785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519078041656034:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:01.294628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519077324079666:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.294731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:01.294784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519077324079666:2109] [node 6] ICP22 created new session: [3:7571519081619046998:2109]
2025-11-11T17:21:01.294836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519081619046998:2109] [node 6] ICS09 handshake done sender: [3:7571519081619046970:2114] self: [3:7571519081619046969:0] peer: [6:7571519081670126004:0] socket: 575 qp: -1
2025-11-11T17:21:01.294873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519081619046998:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:01.294906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:01.306261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519081670126006:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51826
2025-11-11T17:21:01.307304Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.309716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519077360414831:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519081655382215:2119] held: no
2025-11-11T17:21:01.309977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519081655382228:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26636
2025-11-11T17:21:01.311003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519077375158610:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519081670125996:2120] held: no
2025-11-11T17:21:01.311163Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:01.312894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519077375158612:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519081670126003:2122] held: no
2025-11-11T17:21:01.313038Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519081670125996:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:01.313386Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519081670126003:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:01.313681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519077375158610:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.313769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:01.313821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519077375158610:2048] [node 1] ICP22 created new session: [6:7571519081670126007:2048]
2025-11-11T17:21:01.313867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519081670126007:2048] [node 1] ICS09 handshake done sender: [6:7571519081670125996:2120] self: [6:7571519081670126002:0] peer: [1:7571519078718823232:0] socket: 602 qp: -1
2025-11-11T17:21:01.313903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519081670126007:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:01.313994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:01.311315Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519078718823235:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:01.311360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519078718823235:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:01.315113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519077375158612:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.315198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:01.315241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519077375158612:2050] [node 3] ICP22 created new session: [6:7571519081670126009:2050]
2025-11-11T17:21:01.315286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519081670126009:2050] [node 3] ICS09 handshake done sender: [6:7571519081670126003:2122] self: [6:7571519081670126004:0] peer: [3:7571519081619046969:0] socket: 608 qp: -1
2025-11-11T17:21:01.315317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519081670126009:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:01.315351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:01.321997Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519081655382215:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:01.323567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519082336623432:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46908
2025-11-11T17:21:01.322418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519077360414831:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.322506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:01.322559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519077360414831:2049] [node 2] ICP22 created new session: [4:7571519081655382229:2049]
2025-11-11T17:21:01.322607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519081655382229:2049] [node 2] ICS09 handshake done sender: [4:7571519081655382215:2119] self: [4:7571519081655382225:0] peer: [2:7571519077688919396:0] socket: 595 qp: -1
2025-11-11T17:21:01.322659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519081655382229:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:01.322689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:01.325789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519078718823235:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:01.326697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519082336623424:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:01.326862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519078718823225:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:01.326975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:01.327034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519078718823225:2109] [node 5] ICP22 created new session: [1:7571519083013790560:2109]
2025-11-11T17:21:01.327085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519083013790560:2109] [node 5] ICS09 handshake done sender: [1:7571519078718823235:2115] self: [1:7571519078718823234:0] peer: [5:7571519082336623431:0] socket: 559 qp: -1
2025-11-11T17:21:01.327121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519083013790560:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:01.327160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:01.327002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519078041656034:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:01.327078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:01.327123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519078041656034:2048] [node 1] ICP22 created new session: [5:7571519082336623433:2048]
2025-11-11T17:21:01.327172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519082336623433:2048] [node 1] ICS09 handshake done sender: [5:7571519082336623424:2121] self: [5:7571519082336623431:0] peer: [1:7571519078718823234:0] socket: 556 qp: -1
2025-11-11T17:21:01.327214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519082336623433:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:01.327245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:01.357111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:01.357371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519081983886711:2048] [node 1] ICS01 socket: 566 reason# EndOfStream
2025-11-11T17:21:01.357420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519073393952028:2048] [node 1] ICP30 unregister session Session# [2:7571519081983886711:2048] VirtualId# [2:7571519081983886707:0]
2025-11-11T17:21:01.357459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519081983886711:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.357557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:01.357621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:01.359474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:01.359687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519081619046990:2048] [node 1] ICS01 socket: 571 reason# EndOfStream
2025-11-11T17:21:01.359723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519073029112303:2048] [node 1] ICP30 unregister session Session# [3:7571519081619046990:2048] VirtualId# [3:7571519081619046984:0]
2025-11-11T17:21:01.359755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519081619046990:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.359827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:01.359868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:01.365834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:01.366054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519081670126007:2048] [node 1] ICS01 socket: 602 reason# EndOfStream
2025-11-11T17:21:01.366096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519077375158610:2048] [node 1] ICP30 unregister session Session# [6:7571519081670126007:2048] VirtualId# [6:7571519081670126002:0]
2025-11-11T17:21:01.366132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519081670126007:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.366212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:01.366259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:01.368105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:01.368364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519082336623433:2048] [node 1] ICS01 socket: 556 reason# EndOfStream
2025-11-11T17:21:01.368427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519078041656034:2048] [node 1] ICP30 unregister session Session# [5:7571519082336623433:2048] VirtualId# [5:7571519082336623431:0]
2025-11-11T17:21:01.368471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519082336623433:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.368566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:01.368624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:01.367227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:01.367528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519081655382222:2048] [node 1] ICS01 socket: 579 reason# EndOfStream
2025-11-11T17:21:01.367610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519077360414830:2048] [node 1] ICP30 unregister session Session# [4:7571519081655382222:2048] VirtualId# [4:7571519081655382214:0]
2025-11-11T17:21:01.367669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519081655382222:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.367831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:01.367904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:01.440695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:01.440916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519081619046988:2049] [node 2] ICS01 socket: 572 reason# EndOfStream
2025-11-11T17:21:01.440949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519073029112304:2049] [node 2] ICP30 unregister session Session# [3:7571519081619046988:2049] VirtualId# [3:7571519081619046983:0]
2025-11-11T17:21:01.440979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519081619046988:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.441071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:01.441107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:01.453764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:01.453784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:01.454053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519081670125994:2049] [node 2] ICS01 socket: 578 reason# EndOfStream
2025-11-11T17:21:01.454113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519081655382229:2049] [node 2] ICS01 socket: 595 reason# EndOfStream
2025-11-11T17:21:01.454159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519077360414831:2049] [node 2] ICP30 unregister session Session# [4:7571519081655382229:2049] VirtualId# [4:7571519081655382225:0]
2025-11-11T17:21:01.454096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519077375158611:2049] [node 2] ICP30 unregister session Session# [6:7571519081670125994:2049] VirtualId# [6:7571519081670125991:0]
2025-11-11T17:21:01.454202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519081655382229:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.454131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519081670125994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.454251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:01.454288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:01.454303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:01.454331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:01.454427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:01.454657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519082336623429:2049] [node 2] ICS07 socket disconnect 547 reason# EndOfStream
2025-11-11T17:21:01.454685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519082336623429:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.454763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:01.454937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519082336623429:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:01.455608Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Couldn't connect to any resolved address
2025-11-11T17:21:01.455892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519078041656035:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:01.455951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519082336623429:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:01.455986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519078041656035:2049] [node 2] ICP30 unregister session Session# [5:7571519082336623429:2049] VirtualId# [5:7571519082336623427:0]
2025-11-11T17:21:01.456035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:01.456070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519078041656035:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:01.456087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11537) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:01.465713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519078041656035:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:01.517474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:01.517743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519081670126009:2050] [node 3] ICS01 socket: 608 reason# EndOfStream
2025-11-11T17:21:01.517730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:01.517785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519077375158612:2050] [node 3] ICP30 unregister session Session# [6:7571519081670126009:2050] VirtualId# [6:7571519081670126004:0]
2025-11-11T17:21:01.517966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519081655382226:2050] [node 3] ICS07 socket disconnect 591 reason# EndOfStream
2025-11-11T17:21:01.517818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519081670126009:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.517995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519081655382226:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.517909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:01.518106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:01.517955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:01.518154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519081655382226:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:01.518393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:01.518643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519082336623415:2050] [node 3] ICS01 socket: 576 reason# EndOfStream
2025-11-11T17:21:01.518707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519078041656036:2050] [node 3] ICP30 unregister session Session# [5:7571519082336623415:2050] VirtualId# [5:7571519082336623411:0]
2025-11-11T17:21:01.518756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519082336623415:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.518852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:01.518914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:01.519159Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Couldn't connect to any resolved address
2025-11-11T17:21:01.523791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519077360414832:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:01.523856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519081655382226:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:01.523898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519077360414832:2050] [node 3] ICP30 unregister session Session# [4:7571519081655382226:2050] VirtualId# [4:7571519081655382221:0]
2025-11-11T17:21:01.523968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:01.524001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519077360414832:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:01.524040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26840) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:01.553179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519077360414832:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:01.609750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:01.609991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519082336623420:2067] [node 4] ICS01 socket: 570 reason# EndOfStream
2025-11-11T17:21:01.610036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519078041656099:2067] [node 4] ICP30 unregister session Session# [5:7571519082336623420:2067] VirtualId# [5:7571519082336623417:0]
2025-11-11T17:21:01.610073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519082336623420:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.610181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:01.610238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:01.611248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:01.611509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519081670125988:2069] [node 4] ICS01 socket: 558 reason# EndOfStream
2025-11-11T17:21:01.611560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519077375158674:2069] [node 4] ICP30 unregister session Session# [6:7571519081670125988:2069] VirtualId# [6:7571519081670125986:0]
2025-11-11T17:21:01.611594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519081670125988:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.611688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:01.611738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:01.643866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:01.644129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519081670126000:2065] [node 5] ICS01 socket: 582 reason# EndOfStream
2025-11-11T17:21:01.644181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519077375158629:2065] [node 5] ICP30 unregister session Session# [6:7571519081670126000:2065] VirtualId# [6:7571519081670125998:0]
2025-11-11T17:21:01.644224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519081670126000:2065] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:01.644319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:01.644365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:01.746927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519082940944370:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:01.746946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519082940944371:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:01.771149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519084943126142:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:01.771193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519084943126143:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:01.796085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519083276333974:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:01.796314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519083276333973:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:01.819395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519083648158796:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:01.819440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519083648158797:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:01.819481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519083648158798:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:01.843077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519081644614982:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:01.843115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519081644614983:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:01.843194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519081644614984:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:01.878935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519084656497254:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:01.878982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519084656497255:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:01.879013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519084656497256:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:02.164756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519088951464606:2101] [node 5] ICP01 ready to work
2025-11-11T17:21:02.238191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519089238093503:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:02.258773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519089238093502:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:02.283070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519087943126164:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:02.299803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519083276333974:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(173087371ns) {Configure:104(173082321ns) {ProcessConfigured:135(88344ns) {StartInitialHandshake:157(85531ns) {DropHandshakes:469(10225ns) {DropIncomingHandshake:441(827ns) DropOutgoingHandshake:456(496ns)} GenerateSessionVirtualId:988(1273ns) PrepareNewSessionHandshake:476(1642ns) {SwitchToState:178(558ns)}}}}}
2025-11-11T17:21:02.293005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519089238093510:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:02.300306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519087571301361:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:02.322756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519088951464613:2105] [node 4] ICP01 ready to work
2025-11-11T17:21:02.322553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519085939582341:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:02.300673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519087571301362:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.342686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519082940944370:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(235201291ns) {Configure:104(235196930ns) {ProcessConfigured:135(92323ns) {StartInitialHandshake:157(90249ns) {DropHandshakes:469(2485ns) {DropIncomingHandshake:441(516ns) DropOutgoingHandshake:456(355ns)} GenerateSessionVirtualId:988(889ns) PrepareNewSessionHandshake:476(5006ns) {SwitchToState:178(832ns)}}}}}
2025-11-11T17:21:02.301006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519087571301363:2123] [node 6] ICP01 ready to work
2025-11-11T17:21:02.361786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519085939582342:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:02.367433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519083648158796:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(129794381ns) {Configure:104(129789291ns) {ProcessConfigured:135(84004ns) {StartInitialHandshake:157(78016ns) {DropHandshakes:469(3026ns) {DropIncomingHandshake:441(778ns) DropOutgoingHandshake:456(334ns)} GenerateSessionVirtualId:988(1460ns) PrepareNewSessionHandshake:476(2083ns) {SwitchToState:178(705ns)}}}}}
2025-11-11T17:21:02.343094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519087235911731:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.417344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519089238093516:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53339
2025-11-11T17:21:02.387519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519081644614983:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(221533678ns) {Configure:104(221528542ns) {ProcessConfigured:135(74676ns) {StartInitialHandshake:157(68381ns) {DropHandshakes:469(3477ns) {DropIncomingHandshake:441(675ns) DropOutgoingHandshake:456(544ns)} GenerateSessionVirtualId:988(1193ns) PrepareNewSessionHandshake:476(1523ns) {SwitchToState:178(399ns)}}}}}
2025-11-11T17:21:02.394145Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519084656497254:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(91788722ns) {Configure:104(91784663ns) {ProcessConfigured:135(100664ns) {StartInitialHandshake:157(95640ns) {DropHandshakes:469(2900ns) {DropIncomingHandshake:441(632ns) DropOutgoingHandshake:456(410ns)} GenerateSessionVirtualId:988(898ns) PrepareNewSessionHandshake:476(1888ns) {SwitchToState:178(682ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.367963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519087943126167:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:02.380750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519083276333973:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(200469247ns) {Configure:104(200463733ns) {ProcessConfigured:135(19830417ns) {StartInitialHandshake:157(19827084ns) {DropHandshakes:469(3013ns) {DropIncomingHandshake:441(646ns) DropOutgoingHandshake:456(546ns)} GenerateSessionVirtualId:988(846ns) PrepareNewSessionHandshake:476(4024ns) {SwitchToState:178(1480ns)}}}}}
2025-11-11T17:21:02.343378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519087235911732:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:02.435391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519087571301362:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(54385651ns) {Configure:104(54380305ns) {ProcessConfigured:135(76079ns) {StartInitialHandshake:157(70583ns) {DropHandshakes:469(3043ns) {DropIncomingHandshake:441(721ns) DropOutgoingHandshake:456(366ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:476(1556ns) {SwitchToState:178(542ns)}}}}}
2025-11-11T17:21:02.343630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519087235911733:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.469919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519081644614982:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(84048070ns) {Configure:104(84036375ns) {ProcessConfigured:135(10085433ns) {StartInitialHandshake:157(10082458ns) {DropHandshakes:469(3421ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(725ns)} GenerateSessionVirtualId:988(1101ns) PrepareNewSessionHandshake:476(2734ns) {SwitchToState:178(509ns)}}}}}
2025-11-11T17:21:02.421945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519083648158797:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(53604975ns) {Configure:104(53594217ns) {ProcessConfigured:135(16781283ns) {StartInitialHandshake:157(16777886ns) {DropHandshakes:469(1634031ns) {DropIncomingHandshake:441(589ns) DropOutgoingHandshake:456(2192ns)} GenerateSessionVirtualId:988(2675ns) PrepareNewSessionHandshake:476(6859ns) {SwitchToState:178(2287ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.475023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519089238093517:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53339
2025-11-11T17:21:02.475094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519089238093518:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53339
2025-11-11T17:21:02.484146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093507:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.484187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519089238093507:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.378609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519087235911733:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17925042ns) {Configure:104(17921466ns) {ProcessConfigured:135(69213ns) {StartInitialHandshake:157(66811ns) {DropHandshakes:469(2928ns) {DropIncomingHandshake:441(727ns) DropOutgoingHandshake:456(546ns)} GenerateSessionVirtualId:988(705ns) PrepareNewSessionHandshake:476(1449ns) {SwitchToState:178(578ns)}}}}}
2025-11-11T17:21:02.427928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519087943126172:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21310
2025-11-11T17:21:02.482950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519088951464606:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(152899543ns) {Configure:104(152892904ns) {ProcessConfigured:135(73050ns) {StartInitialHandshake:157(70192ns) {DropHandshakes:469(2645ns) {DropIncomingHandshake:441(621ns) DropOutgoingHandshake:456(510ns)} GenerateSessionVirtualId:988(828ns) PrepareNewSessionHandshake:476(1799ns) {SwitchToState:178(456ns)}}}}}
2025-11-11T17:21:02.485298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519084656497255:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(88970480ns) {Configure:104(88965624ns) {ProcessConfigured:135(23213965ns) {StartInitialHandshake:157(23209970ns) {DropHandshakes:469(2381ns) {DropIncomingHandshake:441(524ns) DropOutgoingHandshake:456(405ns)} GenerateSessionVirtualId:988(1274ns) PrepareNewSessionHandshake:476(2960ns) {SwitchToState:178(383ns)}}}}}
2025-11-11T17:21:02.379062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519087235911734:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18538
2025-11-11T17:21:02.502943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519087571301361:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(201375136ns) {Configure:104(201368836ns) {ProcessConfigured:135(38141404ns) {StartInitialHandshake:157(38139054ns) {DropHandshakes:469(1382ns) {DropIncomingHandshake:441(339ns) DropOutgoingHandshake:456(231ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:476(10704ns) {SwitchToState:178(3219ns)}}}}}
2025-11-11T17:21:02.530030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519081644614984:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(142119288ns) {Configure:104(142114707ns) {ProcessConfigured:135(14817540ns) {StartInitialHandshake:157(14815360ns) {DropHandshakes:469(2777ns) {DropIncomingHandshake:441(599ns) DropOutgoingHandshake:456(420ns)} GenerateSessionVirtualId:988(853ns) PrepareNewSessionHandshake:476(6463ns) {SwitchToState:178(2428ns)}}}}}
2025-11-11T17:21:02.428091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519087943126173:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21310
2025-11-11T17:21:02.503680Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519089238093507:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.569886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519087571301375:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14453
2025-11-11T17:21:02.463176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519083648158798:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(175958499ns) {Configure:104(175955503ns) {ProcessConfigured:135(74346ns) {StartInitialHandshake:157(72349ns) {DropHandshakes:469(4191ns) {DropIncomingHandshake:441(254ns) DropOutgoingHandshake:456(157ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:476(2733ns) {SwitchToState:178(947ns)}}}}}
2025-11-11T17:21:02.543600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519089238093502:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.465723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519082940944371:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(330846547ns) {Configure:104(330842739ns) {ProcessConfigured:135(20180872ns) {StartInitialHandshake:157(20178417ns) {DropHandshakes:469(3390ns) {DropIncomingHandshake:441(813ns) DropOutgoingHandshake:456(549ns)} GenerateSessionVirtualId:988(903ns) PrepareNewSessionHandshake:476(3847ns) {SwitchToState:178(1663ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.463427Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519087943126172:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.469254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519087235911746:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18538
2025-11-11T17:21:02.543677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.590261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519084656497256:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(107001634ns) {Configure:104(106995934ns) {ProcessConfigured:135(87904ns) {StartInitialHandshake:157(84834ns) {DropHandshakes:469(2432ns) {DropIncomingHandshake:441(424ns) DropOutgoingHandshake:456(549ns)} GenerateSessionVirtualId:988(871ns) PrepareNewSessionHandshake:476(2388ns) {SwitchToState:178(515ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.543733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519089238093502:2107] [node 4] ICP22 created new session: [2:7571519089238093521:2107]
2025-11-11T17:21:02.604480Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301375:2142] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.615795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519083276333974:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.616074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519087571301376:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14453
2025-11-11T17:21:02.623393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519087571301378:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14453
2025-11-11T17:21:02.625286Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519087571301375:2142] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:02.599527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519085939582342:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(69152505ns) {Configure:104(69147084ns) {ProcessConfigured:135(101310ns) {StartInitialHandshake:157(98141ns) {DropHandshakes:469(3203ns) {DropIncomingHandshake:441(936ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(1262ns) PrepareNewSessionHandshake:476(1420ns) {SwitchToState:178(483ns)}}}}}
2025-11-11T17:21:02.463680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519083648158797:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.469349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519087235911747:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18538
2025-11-11T17:21:02.600791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582347:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.634764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519083276333974:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.543792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519089238093521:2107] [node 4] ICS09 handshake done sender: [2:7571519089238093507:2112] self: [2:7571519089238093506:0] peer: [4:7571519087943126181:0] socket: 553 qp: -1
2025-11-11T17:21:02.506184Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519087943126173:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.554789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519083648158796:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.600974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582348:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
2025-11-11T17:21:02.494183Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911734:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.494661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519087235911748:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18538
2025-11-11T17:21:02.562013Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911730:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.562055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519087235911730:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.587586Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911748:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.593223Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911747:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.638332Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911746:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.638543Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911736:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.638573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519087235911736:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.639885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519087235911752:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18538
2025-11-11T17:21:02.661791Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519087235911730:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:02.663941Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519087235911736:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:02.664265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519082940944370:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.664341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:02.664395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519082940944370:2048] [node 2] ICP22 created new session: [1:7571519087235911756:2048]
2025-11-11T17:21:02.664440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519087235911756:2048] [node 2] ICS09 handshake done sender: [1:7571519087235911730:2110] self: [1:7571519087235911729:0] peer: [2:7571519089238093524:0] socket: 556 qp: -1
2025-11-11T17:21:02.664474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519087235911756:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:02.664507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:02.664895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519087235911731:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.664927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:02.664976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519087235911731:2107] [node 5] ICP22 created new session: [1:7571519087235911758:2107]
2025-11-11T17:21:02.665020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519087235911758:2107] [node 5] ICS09 handshake done sender: [1:7571519087235911736:2112] self: [1:7571519087235911735:0] peer: [5:7571519085939582365:0] socket: 557 qp: -1
2025-11-11T17:21:02.665055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519087235911758:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:02.665083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:02.665721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911738:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.665762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519087235911738:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.543832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519089238093521:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:02.645499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.703042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464630:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.703118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464631:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.711163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464632:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.601115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582349:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
2025-11-11T17:21:02.554905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519087943126182:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21310
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.543870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:02.650047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519083276333974:2049] [node 2] ICP22 created new session: [3:7571519087571301379:2049]
2025-11-11T17:21:02.669946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519087571301379:2049] [node 2] ICS09 handshake done sender: [3:7571519087571301375:2142] self: [3:7571519087571301377:0] peer: [2:7571519089238093511:0] socket: 584 qp: -1
2025-11-11T17:21:02.676162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519087571301379:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.676223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:02.609326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519087943126183:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21310
2025-11-11T17:21:02.601215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582350:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
2025-11-11T17:21:02.601413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582356:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
2025-11-11T17:21:02.606933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519085939582341:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(136585054ns) {Configure:104(136579548ns) {ProcessConfigured:135(13303959ns) {StartInitialHandshake:157(13301356ns) {DropHandshakes:469(1689ns) {DropIncomingHandshake:441(284ns) DropOutgoingHandshake:456(515ns)} GenerateSessionVirtualId:988(627ns) PrepareNewSessionHandshake:476(3146ns) {SwitchToState:178(1051ns)}}}}}
2025-11-11T17:21:02.617730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519085939582347:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.625950Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519085939582356:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.634848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519085939582349:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.634963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519085939582350:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.635094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519085939582348:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.635248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519081644614983:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.635561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519085939582341:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.635616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519081644614984:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.635665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519081644614982:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.641975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
2025-11-11T17:21:02.658021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582367:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
2025-11-11T17:21:02.734534Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519085939582349:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.758824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519085939582341:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.758920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:02.758972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519085939582341:2108] [node 4] ICP22 created new session: [5:7571519085939582370:2108]
2025-11-11T17:21:02.759022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519085939582370:2108] [node 4] ICS09 handshake done sender: [5:7571519085939582349:2114] self: [5:7571519085939582363:0] peer: [4:7571519087943126170:0] socket: 573 qp: -1
2025-11-11T17:21:02.759062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519085939582370:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:02.759121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:02.759750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582368:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.762926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519085939582369:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37681
2025-11-11T17:21:02.763373Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519085939582348:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:02.544965Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093516:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.745721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911752:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.676773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519083276333974:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(44836743ns) {HandleHandshakeStatus:329(44833559ns) {DropIncomingHandshake:441(2174ns) DropOutgoingHandshake:456(16011ns) SwitchToState:178(965ns) ProcessPendingSessionEvents:495(108424ns) {ForwardSessionEventToSession:612(57150ns) ForwardSessionEventToSession:612(6200ns) ForwardSessionEventToSession:612(8574ns) ForwardSessionEventToSession:612(3257ns) ForwardSessionEventToSession:612(2394ns) ForwardSessionEventToSession:612(5107ns) ForwardSessionEventToSession:612(3679ns) ForwardSessionEventToSession:612(5305ns) ForwardSessionEventToSession:612(3265ns) ForwardSessionEventToSession:612(5536ns)}}}
2025-11-11T17:21:02.741966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464629:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.759986Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519088951464630:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.760085Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519088951464629:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.760222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519084656497255:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.760297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519088951464613:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.787836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519087235911731:2107] [node 5] ICP18 handshake (actor [1:7571519087235911752:2119]) is too old
2025-11-11T17:21:02.616915Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.633962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519087943126172:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:02.654140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519083648158797:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519087943126172:2122] held: no
2025-11-11T17:21:02.654431Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519087943126183:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.654551Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519087943126171:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.654573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519087943126171:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.655835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519083648158797:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.655910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:02.655971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519083648158797:2049] [node 2] ICP22 created new session: [4:7571519087943126188:2049]
2025-11-11T17:21:02.656019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519087943126188:2049] [node 2] ICS09 handshake done sender: [4:7571519087943126172:2122] self: [4:7571519087943126181:0] peer: [2:7571519089238093506:0] socket: 560 qp: -1
2025-11-11T17:21:02.656056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519087943126188:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:02.656110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:02.657094Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519087943126171:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:02.657340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519087943126164:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.657392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:02.657440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519087943126164:2108] [node 5] ICP22 created new session: [4:7571519087943126190:2108]
2025-11-11T17:21:02.657485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519087943126190:2108] [node 5] ICS09 handshake done sender: [4:7571519087943126171:2121] self: [4:7571519087943126170:0] peer: [5:7571519085939582363:0] socket: 564 qp: -1
2025-11-11T17:21:02.657517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519087943126190:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:02.657567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:02.722874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519087943126193:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21310
2025-11-11T17:21:02.790926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519087943126193:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.791628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519083648158798:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.791721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519087943126194:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21310
2025-11-11T17:21:02.793896Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519087943126175:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.793929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519087943126175:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.793695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519088951464632:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.545277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519084943126142:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.763902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519081644614982:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.676943Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301367:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.763985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.676967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519087571301367:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.805779Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519087943126175:2124] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:02.803857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519084656497256:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.550183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519089238093510:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(220660182ns) {Configure:104(220651751ns) {ProcessConfigured:135(15121275ns) {StartInitialHandshake:157(15116564ns) {DropHandshakes:469(3654ns) {DropIncomingHandshake:441(1208ns) DropOutgoingHandshake:456(502ns)} GenerateSessionVirtualId:988(2723ns) PrepareNewSessionHandshake:476(2857ns) {SwitchToState:178(547ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.825713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464637:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.764030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519081644614982:2048] [node 1] ICP22 created new session: [5:7571519085939582372:2048]
2025-11-11T17:21:02.684694Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:02.561942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093517:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.833829Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519087235911738:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:02.605756Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093518:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.606284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519089238093502:2107] [node 4] ICP18 handshake (actor [2:7571519089238093517:2116]) is too old
2025-11-11T17:21:02.622021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519089238093526:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53339
2025-11-11T17:21:02.622148Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093512:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.622169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519089238093512:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.709115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519087571301381:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14453
2025-11-11T17:21:02.709336Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519087571301367:2139] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:02.709667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519087571301362:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.709747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:02.709802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519087571301362:2113] [node 5] ICP22 created new session: [3:7571519087571301385:2113]
2025-11-11T17:21:02.709848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519087571301385:2113] [node 5] ICS09 handshake done sender: [3:7571519087571301367:2139] self: [3:7571519087571301366:0] peer: [5:7571519085939582364:0] socket: 567 qp: -1
2025-11-11T17:21:02.709883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519087571301385:2113] [node 5] ICS10 traffic start
2025-11-11T17:21:02.709926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:02.710384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519087571301383:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14453
2025-11-11T17:21:02.761058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301381:2145] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.761160Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301376:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.761435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519083276333974:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:02.761755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519087571301362:2113] [node 5] ICP18 handshake (actor [3:7571519087571301376:2143]) is too old
2025-11-11T17:21:02.805638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301374:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.805687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519087571301374:2141] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.816250Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519087571301374:2141] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:02.833929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301369:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.833968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519087571301369:2140] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.866068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519087571301361:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.866177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:02.866229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519087571301361:2107] [node 4] ICP22 created new session: [3:7571519087571301390:2107]
2025-11-11T17:21:02.866290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519087571301390:2107] [node 4] ICS09 handshake done sender: [3:7571519087571301374:2141] self: [3:7571519087571301373:0] peer: [4:7571519087943126195:0] socket: 608 qp: -1
2025-11-11T17:21:02.866329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519087571301390:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:02.866373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:02.764077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519085939582372:2048] [node 1] ICS09 handshake done sender: [5:7571519085939582348:2113] self: [5:7571519085939582365:0] peer: [1:7571519087235911735:0] socket: 572 qp: -1
2025-11-11T17:21:02.830138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519087943126197:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21310
2025-11-11T17:21:02.825960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.876361Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519088951464631:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.876475Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519088951464629:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:02.876683Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519088951464630:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:02.623614Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519089238093512:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:02.867175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519087235911732:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.867237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:02.867284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519087235911732:2108] [node 4] ICP22 created new session: [1:7571519087235911763:2108]
2025-11-11T17:21:02.867332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519087235911763:2108] [node 4] ICS09 handshake done sender: [1:7571519087235911738:2113] self: [1:7571519087235911737:0] peer: [4:7571519087943126184:0] socket: 558 qp: -1
2025-11-11T17:21:02.867364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519087235911763:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:02.867403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:02.846290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519087943126167:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.764116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519085939582372:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:02.624075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519084943126143:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.624148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:02.624195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519084943126143:2049] [node 3] ICP22 created new session: [2:7571519089238093529:2049]
2025-11-11T17:21:02.624251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519089238093529:2049] [node 3] ICS09 handshake done sender: [2:7571519089238093512:2114] self: [2:7571519089238093511:0] peer: [3:7571519087571301377:0] socket: 583 qp: -1
2025-11-11T17:21:02.624289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519089238093529:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:02.624332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:02.661903Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093509:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.661947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519089238093509:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.685781Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519089238093509:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:02.706919Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:02.735291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519089238093503:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.735367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:02.735420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519089238093503:2109] [node 5] ICP22 created new session: [2:7571519089238093534:2109]
2025-11-11T17:21:02.735477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519089238093534:2109] [node 5] ICS09 handshake done sender: [2:7571519089238093509:2113] self: [2:7571519089238093508:0] peer: [5:7571519085939582361:0] socket: 554 qp: -1
2025-11-11T17:21:02.735513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519089238093534:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:02.735562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:02.736038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519084943126142:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519089238093516:2115] held: no
2025-11-11T17:21:02.736433Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093526:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.736530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519089238093533:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53339
2025-11-11T17:21:02.769957Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519089238093525:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.769999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519089238093525:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.771028Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519089238093516:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:02.785934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519089238093503:2109] [node 5] ICP18 handshake (actor [2:7571519089238093526:2119]) is too old
2025-11-11T17:21:02.786012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519084943126142:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.786063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:02.786110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519084943126142:2048] [node 1] ICP22 created new session: [2:7571519089238093537:2048]
2025-11-11T17:21:02.786163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519089238093537:2048] [node 1] ICS09 handshake done sender: [2:7571519089238093516:2115] self: [2:7571519089238093524:0] peer: [1:7571519087235911729:0] socket: 559 qp: -1
2025-11-11T17:21:02.786201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519089238093537:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:02.786245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:02.804294Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519089238093525:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:02.830526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519089238093510:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.830574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:02.830629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519089238093510:2110] [node 6] ICP22 created new session: [2:7571519089238093539:2110]
2025-11-11T17:21:02.830675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519089238093539:2110] [node 6] ICS09 handshake done sender: [2:7571519089238093525:2118] self: [2:7571519089238093520:0] peer: [6:7571519088951464633:0] socket: 599 qp: -1
2025-11-11T17:21:02.830718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519089238093539:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:02.830781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.887659Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15955) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:02.764181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:02.933735Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911740:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.933814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519087235911740:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.935683Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519087235911740:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:02.846370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:02.888449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519084656497255:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.846425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519087943126167:2109] [node 6] ICP22 created new session: [4:7571519087943126198:2109]
2025-11-11T17:21:02.773647Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519085939582347:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:02.926898Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301383:2146] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.888544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:02.888595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519084656497255:2049] [node 2] ICP22 created new session: [6:7571519088951464638:2049]
2025-11-11T17:21:02.888641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519088951464638:2049] [node 2] ICS09 handshake done sender: [6:7571519088951464630:2117] self: [6:7571519088951464633:0] peer: [2:7571519089238093520:0] socket: 603 qp: -1
2025-11-11T17:21:02.888683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519088951464638:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:02.888742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:02.903281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519088951464613:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.903404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:02.903477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519088951464613:2105] [node 4] ICP22 created new session: [6:7571519088951464640:2105]
2025-11-11T17:21:02.903526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519088951464640:2105] [node 4] ICS09 handshake done sender: [6:7571519088951464629:2116] self: [6:7571519088951464634:0] peer: [4:7571519087943126174:0] socket: 597 qp: -1
2025-11-11T17:21:02.903568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519088951464640:2105] [node 4] ICS10 traffic start
2025-11-11T17:21:02.903611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:02.903886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519088951464613:2105] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:02.904404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519084656497254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.905314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464642:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.935460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464645:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.936289Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519088951464631:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:02.941992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519084656497254:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.942079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:02.942129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519084656497254:2048] [node 1] ICP22 created new session: [6:7571519088951464646:2048]
2025-11-11T17:21:02.942193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519088951464646:2048] [node 1] ICS09 handshake done sender: [6:7571519088951464631:2118] self: [6:7571519088951464643:0] peer: [1:7571519087235911739:0] socket: 605 qp: -1
2025-11-11T17:21:02.942229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519088951464646:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:02.942281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:02.943017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519087235911733:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.943085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:02.846483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519087943126198:2109] [node 6] ICS09 handshake done sender: [4:7571519087943126175:2124] self: [4:7571519087943126174:0] peer: [6:7571519088951464634:0] socket: 595 qp: -1
2025-11-11T17:21:02.799006Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519085939582350:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.943132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519087235911733:2109] [node 6] ICP22 created new session: [1:7571519087235911766:2109]
2025-11-11T17:21:02.961916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519087571301393:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14453
2025-11-11T17:21:02.962104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519083276333973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.962836Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519087571301369:2140] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:02.955982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519088951464642:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.807559Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:02.808691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519081644614983:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.808782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:02.808831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519081644614983:2049] [node 2] ICP22 created new session: [5:7571519085939582374:2049]
2025-11-11T17:21:02.808887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519085939582374:2049] [node 2] ICS09 handshake done sender: [5:7571519085939582347:2112] self: [5:7571519085939582361:0] peer: [2:7571519089238093508:0] socket: 571 qp: -1
2025-11-11T17:21:02.808922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519085939582374:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:02.808997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:02.814291Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:02.854028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519081644614983:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:02.854139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519081644614984:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.881913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:02.893900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519081644614984:2050] [node 3] ICP22 created new session: [5:7571519085939582377:2050]
2025-11-11T17:21:02.893972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519085939582377:2050] [node 3] ICS09 handshake done sender: [5:7571519085939582350:2115] self: [5:7571519085939582364:0] peer: [3:7571519087571301366:0] socket: 574 qp: -1
2025-11-11T17:21:02.894032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519085939582377:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:02.894082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:02.894545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519081644614984:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(47548457ns) {HandleHandshakeStatus:329(47546058ns) {DropIncomingHandshake:441(2398ns) DropOutgoingHandshake:456(9664ns) SwitchToState:178(613ns) ProcessPendingSessionEvents:495(110090ns) {ForwardSessionEventToSession:612(55260ns) ForwardSessionEventToSession:612(3727ns) ForwardSessionEventToSession:612(6983ns) ForwardSessionEventToSession:612(4167ns) ForwardSessionEventToSession:612(6672ns) ForwardSessionEventToSession:612(2927ns) ForwardSessionEventToSession:612(7535ns) ForwardSessionEventToSession:612(8475ns) ForwardSessionEventToSession:612(2677ns) ForwardSessionEventToSession:612(3140ns)}}}
2025-11-11T17:21:02.894843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519081644614984:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:02.969487Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519085939582355:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.969516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519085939582355:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.846521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519087943126198:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:02.965766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519088951464606:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:02.966280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464648:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.981991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519087571301393:2147] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.943175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519087235911766:2109] [node 6] ICS09 handshake done sender: [1:7571519087235911740:2114] self: [1:7571519087235911739:0] peer: [6:7571519088951464643:0] socket: 555 qp: -1
2025-11-11T17:21:02.846843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:02.981780Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519085939582355:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:02.995226Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519088951464632:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:02.943209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519087235911766:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:02.985607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519087571301363:2123] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.853949Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519087943126193:2128] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:02.985682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519085939582342:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:03.002980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519084656497256:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:03.003102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:03.003146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519084656497256:2050] [node 3] ICP22 created new session: [6:7571519093246431947:2050]
2025-11-11T17:21:03.003206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519093246431947:2050] [node 3] ICS09 handshake done sender: [6:7571519088951464632:2119] self: [6:7571519088951464635:0] peer: [3:7571519087571301368:0] socket: 609 qp: -1
2025-11-11T17:21:03.003244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519093246431947:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:03.003274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:02.874998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519087943126167:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(28465241ns) {HandleHandshakeStatus:329(28462754ns) {DropIncomingHandshake:441(25933ns) DropOutgoingHandshake:456(789ns) SwitchToState:178(668ns) ProcessPendingSessionEvents:495(6405275ns) {ForwardSessionEventToSession:612(6349452ns) ForwardSessionEventToSession:612(10050ns) ForwardSessionEventToSession:612(5906ns) ForwardSessionEventToSession:612(5629ns) ForwardSessionEventToSession:612(3004ns) ForwardSessionEventToSession:612(3442ns) ForwardSessionEventToSession:612(5678ns) ForwardSessionEventToSession:612(3617ns) ForwardSessionEventToSession:612(2360ns) ForwardSessionEventToSession:612(4999ns)}}}
2025-11-11T17:21:02.875153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519083648158798:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.875225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:02.875280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519083648158798:2050] [node 3] ICP22 created new session: [4:7571519087943126200:2050]
2025-11-11T17:21:02.875322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519087943126200:2050] [node 3] ICS09 handshake done sender: [4:7571519087943126193:2128] self: [4:7571519087943126195:0] peer: [3:7571519087571301373:0] socket: 610 qp: -1
2025-11-11T17:21:02.875363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519087943126200:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:02.875396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:02.882757Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519087943126173:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:02.925916Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:02.958337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519083648158796:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:02.958448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:02.958519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519083648158796:2048] [node 1] ICP22 created new session: [4:7571519087943126202:2048]
2025-11-11T17:21:02.958576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519087943126202:2048] [node 1] ICS09 handshake done sender: [4:7571519087943126173:2123] self: [4:7571519087943126184:0] peer: [1:7571519087235911737:0] socket: 561 qp: -1
2025-11-11T17:21:02.958622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519087943126202:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:02.958659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:02.958993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519083648158796:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:03.009363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519088951464650:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2392
2025-11-11T17:21:02.943238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:02.985676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:02.985732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519087571301363:2123] [node 6] ICP22 created new session: [3:7571519087571301395:2123]
2025-11-11T17:21:02.996496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519087571301396:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14453
2025-11-11T17:21:02.996848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519087571301395:2123] [node 6] ICS09 handshake done sender: [3:7571519087571301369:2140] self: [3:7571519087571301368:0] peer: [6:7571519088951464635:0] socket: 607 qp: -1
2025-11-11T17:21:02.997239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519087571301395:2123] [node 6] ICS10 traffic start
2025-11-11T17:21:02.997436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:03.002086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519087571301363:2123] [node 6] ICP35 event processing took too much time PendingConnection:278(16310381ns) {HandleHandshakeStatus:329(16307466ns) {DropIncomingHandshake:441(341ns) DropOutgoingHandshake:456(506ns) SwitchToState:178(923ns) ProcessPendingSessionEvents:495(115387ns) {ForwardSessionEventToSession:612(62802ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(6551ns) ForwardSessionEventToSession:612(3512ns) ForwardSessionEventToSession:612(9895ns) ForwardSessionEventToSession:612(2929ns) ForwardSessionEventToSession:612(2910ns) ForwardSessionEventToSession:612(5289ns) ForwardSessionEventToSession:612(5106ns) ForwardSessionEventToSession:612(3069ns)}}}
2025-11-11T17:21:03.002240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519087571301363:2123] [node 6] ICP18 handshake (actor [3:7571519087571301393:2147]) is too old
2025-11-11T17:21:03.002544Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519087571301383:2146] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:02.973951Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519087235911745:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:02.973989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519087235911745:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:02.978052Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519087235911745:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:02.997189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519082940944371:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:02.997630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:02.997681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519082940944371:2049] [node 3] ICP22 created new session: [1:7571519087235911769:2049]
2025-11-11T17:21:02.997729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519087235911769:2049] [node 3] ICS09 handshake done sender: [1:7571519087235911745:2115] self: [1:7571519087235911744:0] peer: [3:7571519087571301394:0] socket: 604 qp: -1
2025-11-11T17:21:02.997762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519087235911769:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:02.997799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:02.985790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:02.989678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519085939582342:2109] [node 6] ICP22 created new session: [5:7571519085939582380:2109]
2025-11-11T17:21:02.989773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519085939582380:2109] [node 6] ICS09 handshake done sender: [5:7571519085939582355:2117] self: [5:7571519085939582354:0] peer: [6:7571519088951464649:0] socket: 568 qp: -1
2025-11-11T17:21:02.989818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519085939582380:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:02.989869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:03.010136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519085939582342:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(17125153ns) {HandleHandshakeStatus:329(17118877ns) {DropIncomingHandshake:441(18032ns) DropOutgoingHandshake:456(322ns) SwitchToState:178(431ns) ProcessPendingSessionEvents:495(12771793ns) {ForwardSessionEventToSession:612(52339ns) ForwardSessionEventToSession:612(4086614ns) ForwardSessionEventToSession:612(2531091ns) ForwardSessionEventToSession:612(4259ns) ForwardSessionEventToSession:612(3549ns) ForwardSessionEventToSession:612(493105ns) ForwardSessionEventToSession:612(2878ns) ForwardSessionEventToSession:612(961088ns) ForwardSessionEventToSession:612(3930ns) ForwardSessionEventToSession:612(4605293ns)}}}
2025-11-11T17:21:03.017751Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519088951464642:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:03.018048Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12691) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:03.023414Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:03.026120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519083276333973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:03.026190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:03.026249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519083276333973:2048] [node 1] ICP22 created new session: [3:7571519091866268694:2048]
2025-11-11T17:21:03.026295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519091866268694:2048] [node 1] ICS09 handshake done sender: [3:7571519087571301383:2146] self: [3:7571519087571301394:0] peer: [1:7571519087235911744:0] socket: 565 qp: -1
2025-11-11T17:21:03.026336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519091866268694:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:03.026375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:03.026650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519083276333973:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:03.022203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519088951464606:2101] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12691) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519088951464642:2122] held: no
2025-11-11T17:21:03.028339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519088951464606:2101] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:03.029847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:03.029917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519088951464606:2101] [node 5] ICP22 created new session: [6:7571519093246431949:2101]
2025-11-11T17:21:03.029977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519093246431949:2101] [node 5] ICS09 handshake done sender: [6:7571519088951464642:2122] self: [6:7571519088951464649:0] peer: [5:7571519085939582354:0] socket: 578 qp: -1
2025-11-11T17:21:03.030019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519093246431949:2101] [node 5] ICS10 traffic start
2025-11-11T17:21:03.030054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:03.102369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:03.102603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519089238093537:2048] [node 1] ICS01 socket: 559 reason# EndOfStream
2025-11-11T17:21:03.102643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519084943126142:2048] [node 1] ICP30 unregister session Session# [2:7571519089238093537:2048] VirtualId# [2:7571519089238093524:0]
2025-11-11T17:21:03.102677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519089238093537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.102753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:03.102794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:03.104776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:03.105006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519091866268694:2048] [node 1] ICS01 socket: 565 reason# EndOfStream
2025-11-11T17:21:03.105049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519083276333973:2048] [node 1] ICP30 unregister session Session# [3:7571519091866268694:2048] VirtualId# [3:7571519087571301394:0]
2025-11-11T17:21:03.105082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519091866268694:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.105147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:03.105181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:03.113778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:03.114096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519088951464646:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-11-11T17:21:03.114149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519084656497254:2048] [node 1] ICP30 unregister session Session# [6:7571519088951464646:2048] VirtualId# [6:7571519088951464643:0]
2025-11-11T17:21:03.114187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519088951464646:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.114281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:03.114330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:03.114441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:03.114622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519087943126202:2048] [node 1] ICS07 socket disconnect 561 reason# EndOfStream
2025-11-11T17:21:03.114648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519087943126202:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.114710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:03.114754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519087943126202:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:03.119609Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Couldn't connect to any resolved address
2025-11-11T17:21:03.119919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519083648158796:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:03.119975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519087943126202:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:03.120010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519083648158796:2048] [node 1] ICP30 unregister session Session# [4:7571519087943126202:2048] VirtualId# [4:7571519087943126184:0]
2025-11-11T17:21:03.120059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:03.113880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:03.114098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519085939582372:2048] [node 1] ICS01 socket: 572 reason# EndOfStream
2025-11-11T17:21:03.114162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519081644614982:2048] [node 1] ICP30 unregister session Session# [5:7571519085939582372:2048] VirtualId# [5:7571519085939582365:0]
2025-11-11T17:21:03.114201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519085939582372:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.114278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:03.114327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:03.120915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519083648158796:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:03.125757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27208) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:03.135847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519083648158796:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:03.184601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:03.184847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519087571301379:2049] [node 2] ICS01 socket: 584 reason# EndOfStream
2025-11-11T17:21:03.184894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519083276333974:2049] [node 2] ICP30 unregister session Session# [3:7571519087571301379:2049] VirtualId# [3:7571519087571301377:0]
2025-11-11T17:21:03.184934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519087571301379:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.185029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:03.185075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:03.186273Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:03.186523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519087943126188:2049] [node 2] ICS01 socket: 560 reason# EndOfStream
2025-11-11T17:21:03.186571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519083648158797:2049] [node 2] ICP30 unregister session Session# [4:7571519087943126188:2049] VirtualId# [4:7571519087943126181:0]
2025-11-11T17:21:03.186607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519087943126188:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.186743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:03.186811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:03.187115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:03.187304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519085939582374:2049] [node 2] ICS01 socket: 571 reason# EndOfStream
2025-11-11T17:21:03.187353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519081644614983:2049] [node 2] ICP30 unregister session Session# [5:7571519085939582374:2049] VirtualId# [5:7571519085939582361:0]
2025-11-11T17:21:03.187385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519085939582374:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.187469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:03.187509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:03.189730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:03.189925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519088951464638:2049] [node 2] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-11T17:21:03.189957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519088951464638:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.190048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:03.190088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519088951464638:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:03.192067Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Couldn't connect to any resolved address
2025-11-11T17:21:03.192310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519084656497255:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:03.192344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519088951464638:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:03.192369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519084656497255:2049] [node 2] ICP30 unregister session Session# [6:7571519088951464638:2049] VirtualId# [6:7571519088951464633:0]
2025-11-11T17:21:03.192403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:03.192431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519084656497255:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:03.192448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:03.200975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519084656497255:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:03.269775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:03.270239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:03.270424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519087943126200:2050] [node 3] ICS01 socket: 610 reason# EndOfStream
2025-11-11T17:21:03.270475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519083648158798:2050] [node 3] ICP30 unregister session Session# [4:7571519087943126200:2050] VirtualId# [4:7571519087943126195:0]
2025-11-11T17:21:03.270511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519087943126200:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.270596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:03.270639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:03.270821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:03.271005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519085939582377:2050] [node 3] ICS01 socket: 574 reason# EndOfStream
2025-11-11T17:21:03.271042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519081644614984:2050] [node 3] ICP30 unregister session Session# [5:7571519085939582377:2050] VirtualId# [5:7571519085939582364:0]
2025-11-11T17:21:03.271066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519085939582377:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.271116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:03.271148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:03.272204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519093246431947:2050] [node 3] ICS01 socket: 609 reason# EndOfStream
2025-11-11T17:21:03.272251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519084656497256:2050] [node 3] ICP30 unregister session Session# [6:7571519093246431947:2050] VirtualId# [6:7571519088951464635:0]
2025-11-11T17:21:03.272280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519093246431947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.272341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:03.272376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:03.373563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:03.373815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519085939582370:2108] [node 4] ICS01 socket: 573 reason# EndOfStream
2025-11-11T17:21:03.373860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519085939582341:2108] [node 4] ICP30 unregister session Session# [5:7571519085939582370:2108] VirtualId# [5:7571519085939582363:0]
2025-11-11T17:21:03.373897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519085939582370:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.373975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:03.374025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:03.377746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:03.377986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519088951464640:2105] [node 4] ICS01 socket: 597 reason# EndOfStream
2025-11-11T17:21:03.378028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519088951464613:2105] [node 4] ICP30 unregister session Session# [6:7571519088951464640:2105] VirtualId# [6:7571519088951464634:0]
2025-11-11T17:21:03.378065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519088951464640:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.378143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:03.378188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:03.441752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:03.441979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519093246431949:2101] [node 5] ICS01 socket: 578 reason# EndOfStream
2025-11-11T17:21:03.442022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519088951464606:2101] [node 5] ICP30 unregister session Session# [6:7571519093246431949:2101] VirtualId# [6:7571519088951464649:0]
2025-11-11T17:21:03.442056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519093246431949:2101] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:03.442165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:03.442209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:03.606543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519090177316369:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:03.606606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519090177316370:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:03.654683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519091920278633:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:03.654723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519091920278634:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:03.691248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519090904164025:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:03.691549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519090904164024:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:03.721956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519089637030770:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:03.722002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519089637030771:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:03.722032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519089637030772:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:03.739244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519091997304967:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:03.739290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519091997304969:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:03.740645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519091997304968:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:03.759154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519091114312777:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:03.759197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519091114312778:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:03.759225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519091114312779:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:04.288619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519095409280099:2072] [node 5] ICP01 ready to work
2025-11-11T17:21:04.301498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519095199131380:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:04.325246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519093931998128:2103] [node 5] ICP01 ready to work
2025-11-11T17:21:04.362054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519096292272319:2100] [node 4] ICP01 ready to work
2025-11-11T17:21:04.434206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519095199131384:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:04.434575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519095199131385:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:04.433798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519096215245992:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:04.444187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519093931998130:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:04.453711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519094472283728:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:04.446284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519095409280125:2074] [node 4] ICP01 ready to work
2025-11-11T17:21:04.442791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519096215245993:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:04.473815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519096292272326:2102] [node 6] ICP01 ready to work
2025-11-11T17:21:04.503382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519091997304967:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(184947010ns) {Configure:104(184941524ns) {ProcessConfigured:135(80503ns) {StartInitialHandshake:157(77777ns) {DropHandshakes:469(2135ns) {DropIncomingHandshake:441(452ns) DropOutgoingHandshake:456(293ns)} GenerateSessionVirtualId:988(1274ns) PrepareNewSessionHandshake:476(1475ns) {SwitchToState:178(359ns)}}}}}
2025-11-11T17:21:04.466165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519094472283730:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:04.474319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519095409280099:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(185397777ns) {Configure:104(185393000ns) {ProcessConfigured:135(71145ns) {StartInitialHandshake:157(68075ns) {DropHandshakes:469(3288ns) {DropIncomingHandshake:441(786ns) DropOutgoingHandshake:456(742ns)} GenerateSessionVirtualId:988(1000ns) PrepareNewSessionHandshake:476(1621ns) {SwitchToState:178(479ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.498597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519096215245994:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:04.470353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519094472283729:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.510659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519095199131380:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(208778730ns) {Configure:104(208773045ns) {ProcessConfigured:135(84492ns) {StartInitialHandshake:157(80980ns) {DropHandshakes:469(4381ns) {DropIncomingHandshake:441(907ns) DropOutgoingHandshake:456(634ns)} GenerateSessionVirtualId:988(1562ns) PrepareNewSessionHandshake:476(1984ns) {SwitchToState:178(723ns)}}}}}
2025-11-11T17:21:04.497779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519094472283728:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29188050ns) {Configure:104(29183742ns) {ProcessConfigured:135(7657037ns) {StartInitialHandshake:157(7654509ns) {DropHandshakes:469(3159ns) {DropIncomingHandshake:441(686ns) DropOutgoingHandshake:456(469ns)} GenerateSessionVirtualId:988(1064ns) PrepareNewSessionHandshake:476(2487ns) {SwitchToState:178(713ns)}}}}}
2025-11-11T17:21:04.519090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519094472283730:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(22508479ns) {Configure:104(22504728ns) {ProcessConfigured:135(96944ns) {StartInitialHandshake:157(93039ns) {DropHandshakes:469(3970ns) {DropIncomingHandshake:441(1116ns) DropOutgoingHandshake:456(641ns)} GenerateSessionVirtualId:988(1297ns) PrepareNewSessionHandshake:476(2942ns) {SwitchToState:178(1167ns)}}}}}
2025-11-11T17:21:04.519485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519094472283734:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-11T17:21:04.558765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519090177316369:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(272725743ns) {Configure:104(267926444ns) {ProcessConfigured:135(82342ns) {StartInitialHandshake:157(75475ns) {DropHandshakes:469(3469ns) {DropIncomingHandshake:441(787ns) DropOutgoingHandshake:456(588ns)} GenerateSessionVirtualId:988(1455ns) PrepareNewSessionHandshake:476(3240ns) {SwitchToState:178(1480ns)}}}}}
2025-11-11T17:21:04.482246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519095409280125:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32545097ns) {Configure:104(32541392ns) {ProcessConfigured:135(95186ns) {StartInitialHandshake:157(92219ns) {DropHandshakes:469(2359ns) {DropIncomingHandshake:441(481ns) DropOutgoingHandshake:456(314ns)} GenerateSessionVirtualId:988(1033ns) PrepareNewSessionHandshake:476(2130ns) {SwitchToState:178(683ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.567718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519094472283745:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-11T17:21:04.545886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519095199131385:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21870634ns) {Configure:104(21866903ns) {ProcessConfigured:135(13870112ns) {StartInitialHandshake:157(13867567ns) {DropHandshakes:469(2458ns) {DropIncomingHandshake:441(547ns) DropOutgoingHandshake:456(708ns)} GenerateSessionVirtualId:988(1006ns) PrepareNewSessionHandshake:476(5322ns) {SwitchToState:178(2877ns)}}}}}
2025-11-11T17:21:04.602564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519089637030772:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(316545355ns) {Configure:104(316533966ns) {ProcessConfigured:135(84739ns) {StartInitialHandshake:157(82051ns) {DropHandshakes:469(3448ns) {DropIncomingHandshake:441(845ns) DropOutgoingHandshake:456(831ns)} GenerateSessionVirtualId:988(1386ns) PrepareNewSessionHandshake:476(2177ns) {SwitchToState:178(757ns)}}}}}
2025-11-11T17:21:04.586196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519096215245993:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(94549155ns) {Configure:104(94537262ns) {ProcessConfigured:135(72507ns) {StartInitialHandshake:157(69703ns) {DropHandshakes:469(3866ns) {DropIncomingHandshake:441(1096ns) DropOutgoingHandshake:456(629ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:476(2357ns) {SwitchToState:178(1031ns)}}}}}
2025-11-11T17:21:04.614955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519096292272319:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(134844854ns) {Configure:104(134834329ns) {ProcessConfigured:135(82883ns) {StartInitialHandshake:157(79677ns) {DropHandshakes:469(4012ns) {DropIncomingHandshake:441(1138ns) DropOutgoingHandshake:456(584ns)} GenerateSessionVirtualId:988(1295ns) PrepareNewSessionHandshake:476(2372ns) {SwitchToState:178(1108ns)}}}}}
2025-11-11T17:21:04.546828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519095199131392:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-11-11T17:21:04.546887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519095199131393:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.618004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519091114312778:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(312877488ns) {Configure:104(312873077ns) {ProcessConfigured:135(12456210ns) {StartInitialHandshake:157(12452741ns) {DropHandshakes:469(2778ns) {DropIncomingHandshake:441(532ns) DropOutgoingHandshake:456(523ns)} GenerateSessionVirtualId:988(713ns) PrepareNewSessionHandshake:476(4630ns) {SwitchToState:178(2491ns)}}}}}
2025-11-11T17:21:04.625703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283734:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.632260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519095199131384:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(197394661ns) {Configure:104(197389820ns) {ProcessConfigured:135(22149250ns) {StartInitialHandshake:157(22147409ns) {DropHandshakes:469(2986ns) {DropIncomingHandshake:441(701ns) DropOutgoingHandshake:456(709ns)} GenerateSessionVirtualId:988(1076ns) PrepareNewSessionHandshake:476(3049ns) {SwitchToState:178(509ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.637939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519089637030771:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(300595273ns) {Configure:104(264120385ns) {ProcessConfigured:135(78932ns) {StartInitialHandshake:157(75793ns) {DropHandshakes:469(3802ns) {DropIncomingHandshake:441(896ns) DropOutgoingHandshake:456(781ns)} GenerateSessionVirtualId:988(1216ns) PrepareNewSessionHandshake:476(2021ns) {SwitchToState:178(592ns)}}}}}
2025-11-11T17:21:04.648942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998137:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.681949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519091997304969:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(222324937ns) {Configure:104(222316671ns) {ProcessConfigured:135(42047446ns) {StartInitialHandshake:157(42043519ns) {DropHandshakes:469(2781ns) {DropIncomingHandshake:441(629ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(469ns) PrepareNewSessionHandshake:476(6084ns) {SwitchToState:178(2708ns)}}}}}
2025-11-11T17:21:04.682896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272335:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.683398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272339:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:04.631394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519096215245992:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(110491528ns) {Configure:104(110485770ns) {ProcessConfigured:135(70694ns) {StartInitialHandshake:157(68039ns) {DropHandshakes:469(3549ns) {DropIncomingHandshake:441(627ns) DropOutgoingHandshake:456(527ns)} GenerateSessionVirtualId:988(1097ns) PrepareNewSessionHandshake:476(2264ns) {SwitchToState:178(601ns)}}}}}
2025-11-11T17:21:04.633899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280149:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:04.688564Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519093931998137:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.646120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519091920278633:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(157037301ns) {Configure:104(157028197ns) {ProcessConfigured:135(143075ns) {StartInitialHandshake:157(136063ns) {DropHandshakes:469(4099ns) {DropIncomingHandshake:441(1001ns) DropOutgoingHandshake:456(707ns)} GenerateSessionVirtualId:988(1487ns) PrepareNewSessionHandshake:476(2171ns) {SwitchToState:178(803ns)}}}}}
2025-11-11T17:21:04.634922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283739:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.634479Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519095409280149:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.683469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272340:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:04.638029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519090904164025:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(335688354ns) {Configure:104(335685191ns) {ProcessConfigured:135(5572312ns) {StartInitialHandshake:157(5567777ns) {DropHandshakes:469(6153ns) {DropIncomingHandshake:441(525ns) DropOutgoingHandshake:456(480ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:476(3825ns) {SwitchToState:178(1911ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.698550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519089637030770:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.634624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519091114312777:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.636223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280153:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:04.637317Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519095409280149:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:04.637552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519091114312777:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:04.637631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:04.637680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519091114312777:2048] [node 1] ICP22 created new session: [6:7571519095409280154:2048]
2025-11-11T17:21:04.637726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519095409280154:2048] [node 1] ICS09 handshake done sender: [6:7571519095409280149:2114] self: [6:7571519095409280152:0] peer: [1:7571519094472283738:0] socket: 565 qp: -1
2025-11-11T17:21:04.637756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519095409280154:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:04.668644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:04.684710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280157:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:04.723530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519091114312777:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(85697015ns) {HandleHandshakeStatus:329(85691353ns) {DropIncomingHandshake:441(995ns) DropOutgoingHandshake:456(14110ns) SwitchToState:178(386ns) ProcessPendingSessionEvents:495(16962545ns) {ForwardSessionEventToSession:612(16898412ns) ForwardSessionEventToSession:612(10989ns) ForwardSessionEventToSession:612(3362ns) ForwardSessionEventToSession:612(2810ns) ForwardSessionEventToSession:612(6388ns) ForwardSessionEventToSession:612(3441ns) ForwardSessionEventToSession:612(3846ns) ForwardSessionEventToSession:612(6655ns) ForwardSessionEventToSession:612(7964ns) ForwardSessionEventToSession:612(5880ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.639417Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519095199131392:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.712982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519091997304968:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(184892399ns) {Configure:104(184887317ns) {ProcessConfigured:135(62222240ns) {StartInitialHandshake:157(62219944ns) {DropHandshakes:469(1882ns) {DropIncomingHandshake:441(173ns) DropOutgoingHandshake:456(713ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(4216ns) {SwitchToState:178(1947ns)}}}}}
2025-11-11T17:21:04.743486Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519096292272335:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.743701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272334:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:04.743855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519091997304969:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.634949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519094472283739:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.698650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998143:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.636458Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519094472283739:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.677798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519094472283730:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.683819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283732:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.683847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519094472283732:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.700366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:04.700416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519094472283730:2109] [node 6] ICP22 created new session: [1:7571519094472283748:2109]
2025-11-11T17:21:04.707704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519094472283748:2109] [node 6] ICS09 handshake done sender: [1:7571519094472283739:2114] self: [1:7571519094472283738:0] peer: [6:7571519095409280152:0] socket: 564 qp: -1
2025-11-11T17:21:04.707758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519094472283748:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:04.707810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:04.708488Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519094472283730:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(39125485ns) {HandleHandshakeStatus:329(39122306ns) {DropIncomingHandshake:441(22971ns) DropOutgoingHandshake:456(640ns) SwitchToState:178(674ns) ProcessPendingSessionEvents:495(132958ns) {ForwardSessionEventToSession:612(59650ns) ForwardSessionEventToSession:612(23313ns) ForwardSessionEventToSession:612(2987ns) ForwardSessionEventToSession:612(7566ns) ForwardSessionEventToSession:612(2566ns) ForwardSessionEventToSession:612(6583ns) ForwardSessionEventToSession:612(2958ns) ForwardSessionEventToSession:612(3228ns) ForwardSessionEventToSession:612(5466ns) ForwardSessionEventToSession:612(4912ns)}}}
2025-11-11T17:21:04.708720Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283736:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.708745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519094472283736:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.709976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519094472283752:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-11T17:21:04.724569Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519094472283732:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:04.742445Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283745:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.753883Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519094472283736:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:04.789927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519090177316370:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.737793Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215245998:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.649801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519095199131393:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.749248Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519096292272339:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.663904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519090904164024:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.737839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519096215245998:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.698825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998144:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.794639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519094472283729:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.803348Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519095409280157:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.698877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998145:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
2025-11-11T17:21:04.699008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998146:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
2025-11-11T17:21:04.726258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519093931998145:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.726385Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519093931998143:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.726621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519089637030772:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.749863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519093931998144:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.775230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519089637030771:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.775331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998150:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
2025-11-11T17:21:04.786327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
2025-11-11T17:21:04.823596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519093931998130:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(325181765ns) {Configure:104(325172603ns) {ProcessConfigured:135(9534686ns) {StartInitialHandshake:157(9531233ns) {DropHandshakes:469(3004ns) {DropIncomingHandshake:441(524ns) DropOutgoingHandshake:456(548ns)} GenerateSessionVirtualId:988(857ns) PrepareNewSessionHandshake:476(2377ns) {SwitchToState:178(919ns)}}}}}
2025-11-11T17:21:04.744680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519096215246008:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17455
2025-11-11T17:21:04.744727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519096215246009:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17455
2025-11-11T17:21:04.744775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519096215246010:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17455
2025-11-11T17:21:04.788957Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519096215245998:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:04.801868Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215245996:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.801917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519096215245996:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.813640Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215246000:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.813695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519096215246000:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.749383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519091997304968:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.670125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519095199131401:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.819727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:04.762180Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519096292272334:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.817938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280158:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:04.853964Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519093931998146:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.670224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519095199131402:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-11-11T17:21:04.856526Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519096215245996:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:04.866695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215246010:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.882063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519091920278634:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.882107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:04.882157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519091920278634:2049] [node 3] ICP22 created new session: [2:7571519096215246014:2049]
2025-11-11T17:21:04.882208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519096215246014:2049] [node 3] ICS09 handshake done sender: [2:7571519096215245998:2111] self: [2:7571519096215245997:0] peer: [3:7571519095199131412:0] socket: 576 qp: -1
2025-11-11T17:21:04.882245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519096215246014:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:04.882276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:04.882791Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215246008:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.883164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519091920278634:2049] [node 3] ICP18 handshake (actor [2:7571519096215246010:2117]) is too old
2025-11-11T17:21:04.828255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519094472283729:2107] [node 4] ICP22 created new session: [1:7571519094472283755:2107]
2025-11-11T17:21:04.822010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519095409280099:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.726911Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519095199131387:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.793859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519096292272340:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.854494Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519093931998134:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.849600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:04.856736Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519095409280158:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.863797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519091114312779:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.866401Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519095409280157:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:04.890628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519095409280099:2072] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:04.890695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:04.890741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519095409280099:2072] [node 5] ICP22 created new session: [6:7571519095409280162:2072]
2025-11-11T17:21:04.890787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519095409280162:2072] [node 5] ICS09 handshake done sender: [6:7571519095409280157:2117] self: [6:7571519095409280159:0] peer: [5:7571519096292272332:0] socket: 591 qp: -1
2025-11-11T17:21:04.890832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519095409280162:2072] [node 5] ICS10 traffic start
2025-11-11T17:21:04.890888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.930764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280164:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:04.830644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283752:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.726937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519095199131387:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.728352Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519095199131387:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:04.728666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519095199131380:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.728709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:04.728756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519095199131380:2107] [node 4] ICP22 created new session: [3:7571519095199131406:2107]
2025-11-11T17:21:04.728805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519095199131406:2107] [node 4] ICS09 handshake done sender: [3:7571519095199131387:2110] self: [3:7571519095199131386:0] peer: [4:7571519093931998148:0] socket: 568 qp: -1
2025-11-11T17:21:04.854524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519093931998134:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.898925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519096215245993:2107] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.958384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280166:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:04.960470Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519095409280164:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.960624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519091114312778:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.812519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519096292272319:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.833950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:04.728835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519095199131406:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.991639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519095409280168:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.898987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.889734Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519093931998143:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.728868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:04.823226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272349:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:04.846967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519090177316370:2049] [node 3] ICP22 created new session: [1:7571519094472283758:2049]
2025-11-11T17:21:05.006120Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:04.825683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519096292272333:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.899037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519096215245993:2107] [node 5] ICP22 created new session: [2:7571519096215246018:2107]
2025-11-11T17:21:04.893898Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519093931998137:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:04.730317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519095199131409:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-11-11T17:21:04.847369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519094472283755:2107] [node 4] ICS09 handshake done sender: [1:7571519094472283736:2112] self: [1:7571519094472283735:0] peer: [4:7571519093931998147:0] socket: 563 qp: -1
2025-11-11T17:21:04.899109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519096215246018:2107] [node 5] ICS09 handshake done sender: [2:7571519096215245996:2110] self: [2:7571519096215245995:0] peer: [5:7571519096292272346:0] socket: 575 qp: -1
2025-11-11T17:21:05.019793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519091114312779:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519095409280158:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.825716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519096292272333:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.734487Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519095199131401:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.734552Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519095199131393:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:04.734840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519090904164024:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:04.734902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:04.734951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519090904164024:2048] [node 1] ICP22 created new session: [3:7571519095199131410:2048]
2025-11-11T17:21:04.734997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519095199131410:2048] [node 1] ICS09 handshake done sender: [3:7571519095199131393:2114] self: [3:7571519095199131404:0] peer: [1:7571519094472283731:0] socket: 567 qp: -1
2025-11-11T17:21:04.735036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519095199131410:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:04.735072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:04.736435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519090904164025:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:04.786811Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519095199131391:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.786855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519095199131391:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.829081Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519095199131391:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:04.833830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519095199131414:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-11-11T17:21:04.862812Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519095199131402:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.862885Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519095199131401:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:04.881686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519095199131385:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.889816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:04.892444Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519095199131398:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.892471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519095199131398:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.892962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519090904164025:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:04.893003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:04.893044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519090904164025:2049] [node 2] ICP22 created new session: [3:7571519095199131417:2049]
2025-11-11T17:21:04.893088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519095199131417:2049] [node 2] ICS09 handshake done sender: [3:7571519095199131401:2117] self: [3:7571519095199131412:0] peer: [2:7571519096215245997:0] socket: 581 qp: -1
2025-11-11T17:21:04.893124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519095199131417:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:04.893158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:04.894154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519095199131385:2109] [node 5] ICP22 created new session: [3:7571519095199131416:2109]
2025-11-11T17:21:04.938294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519095199131380:2107] [node 4] ICP18 handshake (actor [3:7571519095199131402:2118]) is too old
2025-11-11T17:21:04.946611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519095199131416:2109] [node 5] ICS09 handshake done sender: [3:7571519095199131391:2112] self: [3:7571519095199131390:0] peer: [5:7571519096292272345:0] socket: 571 qp: -1
2025-11-11T17:21:04.954288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519095199131416:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:04.954342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:05.002536Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519095199131398:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:05.003112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519095199131384:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:05.003172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:05.003218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519095199131384:2108] [node 6] ICP22 created new session: [3:7571519099494098718:2108]
2025-11-11T17:21:05.003259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519099494098718:2108] [node 6] ICS09 handshake done sender: [3:7571519095199131398:2115] self: [3:7571519095199131397:0] peer: [6:7571519095409280161:0] socket: 569 qp: -1
2025-11-11T17:21:05.003290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519099494098718:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:05.003330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:05.004421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519099494098717:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-11-11T17:21:05.012315Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519099494098717:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:05.031242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519095199131385:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(167714573ns) {HandleHandshakeStatus:329(167711614ns) {DropIncomingHandshake:441(2409ns) DropOutgoingHandshake:456(5855ns) SwitchToState:178(1007ns) ProcessPendingSessionEvents:495(8904276ns) {ForwardSessionEventToSession:612(8850772ns) ForwardSessionEventToSession:612(5276ns) ForwardSessionEventToSession:612(9376ns) ForwardSessionEventToSession:612(3007ns) ForwardSessionEventToSession:612(4764ns) ForwardSessionEventToSession:612(5707ns) ForwardSessionEventToSession:612(3195ns) ForwardSessionEventToSession:612(4916ns) ForwardSessionEventToSession:612(3702ns) ForwardSessionEventToSession:612(2526ns)}}}
2025-11-11T17:21:05.031397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519095199131385:2109] [node 5] ICP18 handshake (actor [3:7571519099494098717:2121]) is too old
2025-11-11T17:21:04.850991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519094472283755:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:04.900040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519089637030772:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:04.851076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:04.863568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519094472283758:2049] [node 3] ICS09 handshake done sender: [1:7571519094472283732:2110] self: [1:7571519094472283731:0] peer: [3:7571519095199131404:0] socket: 562 qp: -1
2025-11-11T17:21:04.887198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519094472283758:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:04.910192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519094472283729:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(115316997ns) {HandleHandshakeStatus:329(115314836ns) {DropIncomingHandshake:441(560ns) DropOutgoingHandshake:456(969ns) SwitchToState:178(727ns) ProcessPendingSessionEvents:495(32174736ns) {ForwardSessionEventToSession:612(32111884ns) ForwardSessionEventToSession:612(7213ns) ForwardSessionEventToSession:612(7202ns) ForwardSessionEventToSession:612(8410ns) ForwardSessionEventToSession:612(2861ns) ForwardSessionEventToSession:612(4980ns) ForwardSessionEventToSession:612(3284ns) ForwardSessionEventToSession:612(3470ns) ForwardSessionEventToSession:612(5564ns) ForwardSessionEventToSession:612(3615ns)}}}
2025-11-11T17:21:04.911298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519094472283760:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-11T17:21:04.925942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:04.938416Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283744:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.938456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519094472283744:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.958100Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:04.994176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519090177316370:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(213088091ns) {HandleHandshakeStatus:329(213085635ns) {DropIncomingHandshake:441(20804ns) DropOutgoingHandshake:456(377ns) SwitchToState:178(511ns) ProcessPendingSessionEvents:495(11939606ns) {ForwardSessionEventToSession:612(11881379ns) ForwardSessionEventToSession:612(12133ns) ForwardSessionEventToSession:612(3535ns) ForwardSessionEventToSession:612(3032ns) ForwardSessionEventToSession:612(9128ns) ForwardSessionEventToSession:612(3205ns) ForwardSessionEventToSession:612(6009ns) ForwardSessionEventToSession:612(2291ns) ForwardSessionEventToSession:612(3764ns) ForwardSessionEventToSession:612(5151ns)}}}
2025-11-11T17:21:04.994301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519094472283762:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-11T17:21:05.002760Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519094472283744:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:05.009957Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283737:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:05.009989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519094472283737:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:05.029960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519090177316369:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:05.030043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:05.030092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519090177316369:2048] [node 2] ICP22 created new session: [1:7571519098767251061:2048]
2025-11-11T17:21:05.030148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519098767251061:2048] [node 2] ICS09 handshake done sender: [1:7571519094472283744:2115] self: [1:7571519094472283743:0] peer: [2:7571519096215246021:0] socket: 597 qp: -1
2025-11-11T17:21:05.030183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519098767251061:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:05.030240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:05.037078Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519094472283737:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:05.075740Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519094472283762:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:05.077681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519094472283728:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:05.077812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:05.077874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519094472283728:2108] [node 5] ICP22 created new session: [1:7571519098767251063:2108]
2025-11-11T17:21:05.077938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519098767251063:2108] [node 5] ICS09 handshake done sender: [1:7571519094472283737:2113] self: [1:7571519094472283733:0] peer: [5:7571519096292272359:0] socket: 613 qp: -1
2025-11-11T17:21:05.077990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519098767251063:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:05.078045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:04.900096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:04.900142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519089637030772:2050] [node 3] ICP22 created new session: [4:7571519093931998156:2050]
2025-11-11T17:21:04.900186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519093931998156:2050] [node 3] ICS09 handshake done sender: [4:7571519093931998143:2115] self: [4:7571519093931998148:0] peer: [3:7571519095199131386:0] socket: 578 qp: -1
2025-11-11T17:21:04.900223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519093931998156:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:04.900252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:04.917872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519089637030770:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:04.949455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:04.954027Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519093931998134:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:04.973721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519089637030770:2048] [node 1] ICP22 created new session: [4:7571519093931998161:2048]
2025-11-11T17:21:04.975895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519093931998161:2048] [node 1] ICS09 handshake done sender: [4:7571519093931998137:2113] self: [4:7571519093931998147:0] peer: [1:7571519094472283735:0] socket: 572 qp: -1
2025-11-11T17:21:04.982051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519093931998155:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36105
2025-11-11T17:21:04.986306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519093931998161:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:05.007201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:05.017192Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12100) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:05.018082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519093931998128:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:05.018198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:05.018258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519093931998128:2103] [node 5] ICP22 created new session: [4:7571519098226965459:2103]
2025-11-11T17:21:05.018306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519098226965459:2103] [node 5] ICS09 handshake done sender: [4:7571519093931998134:2111] self: [4:7571519093931998133:0] peer: [5:7571519096292272348:0] socket: 583 qp: -1
2025-11-11T17:21:05.018343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519098226965459:2103] [node 5] ICS10 traffic start
2025-11-11T17:21:05.018380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:05.018896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519089637030771:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12100) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519093931998144:2116] held: no
2025-11-11T17:21:05.026146Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519093931998144:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:05.036080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519089637030771:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.036171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:05.036238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519089637030771:2049] [node 2] ICP22 created new session: [4:7571519098226965462:2049]
2025-11-11T17:21:05.036305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519098226965462:2049] [node 2] ICS09 handshake done sender: [4:7571519093931998144:2116] self: [4:7571519093931998151:0] peer: [2:7571519096215245999:0] socket: 592 qp: -1
2025-11-11T17:21:05.036345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519098226965462:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:05.036383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:05.066502Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519089637030770:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(148402823ns) {HandleHandshakeStatus:329(148400863ns) {DropIncomingHandshake:441(3776ns) DropOutgoingHandshake:456(14235ns) SwitchToState:178(530ns) ProcessPendingSessionEvents:495(15418146ns) {ForwardSessionEventToSession:612(15365880ns) ForwardSessionEventToSession:612(6211ns) ForwardSessionEventToSession:612(6864ns) ForwardSessionEventToSession:612(3030ns) ForwardSessionEventToSession:612(3334ns) ForwardSessionEventToSession:612(5863ns) ForwardSessionEventToSession:612(4028ns) ForwardSessionEventToSession:612(7667ns) ForwardSessionEventToSession:612(3066ns) ForwardSessionEventToSession:612(2687ns)}}}
2025-11-11T17:21:04.899145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519096215246018:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:04.847050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519096292272319:2100] [node 4] ICP35 event processing took too much time PendingConnection:278(18538875ns) {IncomingHandshake:261(17340509ns) {DropIncomingHandshake:441(184ns) PrepareNewSessionHandshake:476(10605972ns) {SwitchToState:178(8570ns)} GenerateSessionVirtualId:988(1209ns) IssueIncomingHandshakeReply:193(6693634ns) {SwitchToState:178(2006ns)}}}
2025-11-11T17:21:05.093716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519090177316369:2048] [node 2] ICP18 handshake (actor [1:7571519094472283762:2119]) is too old
2025-11-11T17:21:05.102811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [1:7571519090177316369:2048] [node 2] ICP35 event processing took too much time StateWork:293(17813865ns) {IncomingHandshake:261(7104743ns)}
2025-11-11T17:21:05.086589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519091114312779:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(44134089ns) {HandleHandshakeStatus:395(44129271ns) {LogHandshakeFail:472(17375ns) {UpdateErrorStateLog:544(5638ns)} DropOutgoingHandshake:456(3957ns) ProcessPendingSessionEvents:495(28603248ns) {DropSessionEvent:512(5158129ns) DropSessionEvent:512(16937ns) DropSessionEvent:512(5560ns) DropSessionEvent:512(4123ns) DropSessionEvent:512(2697ns) DropSessionEvent:512(4479ns) DropSessionEvent:512(4799ns) DropSessionEvent:512(6326ns) DropSessionEvent:512(3957ns) DropSessionEvent:512(25544ns)}}}
2025-11-11T17:21:04.899180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:05.109463Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519093931998153:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.857001Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519096292272333:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:04.962560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519096292272326:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.962631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:04.962676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519096292272326:2102] [node 6] ICP22 created new session: [5:7571519096292272354:2102]
2025-11-11T17:21:04.962740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519096292272354:2102] [node 6] ICS09 handshake done sender: [5:7571519096292272333:2112] self: [5:7571519096292272332:0] peer: [6:7571519095409280159:0] socket: 590 qp: -1
2025-11-11T17:21:04.962775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519096292272354:2102] [node 6] ICS10 traffic start
2025-11-11T17:21:04.962837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:04.963632Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519096292272339:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:04.965439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272352:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:04.974243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519091997304968:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:04.974329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:04.974379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519091997304968:2049] [node 2] ICP22 created new session: [5:7571519096292272356:2049]
2025-11-11T17:21:04.974423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519096292272356:2049] [node 2] ICS09 handshake done sender: [5:7571519096292272339:2116] self: [5:7571519096292272346:0] peer: [2:7571519096215245995:0] socket: 558 qp: -1
2025-11-11T17:21:04.974464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519096292272356:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:04.974500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:04.990517Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519096292272352:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.991130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519091997304967:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:05.011837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519100587239656:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:05.025785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272353:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:05.035004Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:05.037124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519096292272351:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25407
2025-11-11T17:21:05.065875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519096292272319:2100] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519096292272340:2117] held: no
2025-11-11T17:21:05.067580Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:05.068935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519091997304969:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519096292272335:2114] held: no
2025-11-11T17:21:05.069102Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519096292272340:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:05.074236Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519096292272335:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:05.074548Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519096292272352:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:05.079016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519091997304969:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.079077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:05.079125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519091997304969:2050] [node 3] ICP22 created new session: [5:7571519100587239657:2050]
2025-11-11T17:21:05.079174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519100587239657:2050] [node 3] ICS09 handshake done sender: [5:7571519096292272335:2114] self: [5:7571519096292272345:0] peer: [3:7571519095199131390:0] socket: 574 qp: -1
2025-11-11T17:21:05.079215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519100587239657:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:05.079274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:05.079629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519091997304967:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.079692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:05.079740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519091997304967:2048] [node 1] ICP22 created new session: [5:7571519100587239659:2048]
2025-11-11T17:21:05.079789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519100587239659:2048] [node 1] ICS09 handshake done sender: [5:7571519096292272352:2121] self: [5:7571519096292272359:0] peer: [1:7571519094472283733:0] socket: 615 qp: -1
2025-11-11T17:21:05.079816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519100587239659:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:05.079865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:05.080295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519096292272319:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.080346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:05.080386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519096292272319:2100] [node 4] ICP22 created new session: [5:7571519100587239661:2100]
2025-11-11T17:21:05.080423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519100587239661:2100] [node 4] ICS09 handshake done sender: [5:7571519096292272340:2117] self: [5:7571519096292272348:0] peer: [4:7571519093931998133:0] socket: 587 qp: -1
2025-11-11T17:21:05.080456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519100587239661:2100] [node 4] ICS10 traffic start
2025-11-11T17:21:05.080504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:04.899683Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519096215246000:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:05.091976Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519095409280164:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:04.914023Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215246009:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.916173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519091920278633:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:05.109503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519093931998153:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.959003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519096215245994:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:04.959070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:04.959129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519096215245994:2108] [node 4] ICP22 created new session: [2:7571519096215246023:2108]
2025-11-11T17:21:04.959206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519096215246023:2108] [node 4] ICS09 handshake done sender: [2:7571519096215246000:2112] self: [2:7571519096215245999:0] peer: [4:7571519093931998151:0] socket: 577 qp: -1
2025-11-11T17:21:04.959253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519096215246023:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:04.959291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:04.961447Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215246005:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:04.961478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519096215246005:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:04.975146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519096215246022:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17455
2025-11-11T17:21:05.006973Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519096215246022:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:05.021867Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519096215246005:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:05.058222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519100510213322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17455
2025-11-11T17:21:05.075591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519096215245992:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:05.103427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:05.103509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519096215245992:2109] [node 6] ICP22 created new session: [2:7571519100510213324:2109]
2025-11-11T17:21:05.103565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519100510213324:2109] [node 6] ICS09 handshake done sender: [2:7571519096215246005:2113] self: [2:7571519096215246004:0] peer: [6:7571519095409280167:0] socket: 617 qp: -1
2025-11-11T17:21:05.103620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519100510213324:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:05.103664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:05.107367Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:05.122104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519096215245992:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(28394752ns) {HandleHandshakeStatus:329(28391564ns) {DropIncomingHandshake:441(30983ns) DropOutgoingHandshake:456(585ns) SwitchToState:178(1401ns) ProcessPendingSessionEvents:495(128058ns) {ForwardSessionEventToSession:612(60376ns) ForwardSessionEventToSession:612(8464ns) ForwardSessionEventToSession:612(2789ns) ForwardSessionEventToSession:612(5181ns) ForwardSessionEventToSession:612(5385ns) ForwardSessionEventToSession:612(2375ns) ForwardSessionEventToSession:612(24547ns) ForwardSessionEventToSession:612(2749ns) ForwardSessionEventToSession:612(2254ns) ForwardSessionEventToSession:612(5101ns)}}}
2025-11-11T17:21:05.126279Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519096215246009:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:05.126306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519091920278633:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519096215246009:2116] held: no
2025-11-11T17:21:05.142644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519091920278633:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.142731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:05.142782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519091920278633:2048] [node 1] ICP22 created new session: [2:7571519100510213326:2048]
2025-11-11T17:21:05.142833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519100510213326:2048] [node 1] ICS09 handshake done sender: [2:7571519096215246009:2116] self: [2:7571519096215246021:0] peer: [1:7571519094472283743:0] socket: 604 qp: -1
2025-11-11T17:21:05.142871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519100510213326:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:05.142905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:05.092217Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519095409280158:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:05.092428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519091114312778:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.092498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:05.092543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519091114312778:2049] [node 2] ICP22 created new session: [6:7571519099704247466:2049]
2025-11-11T17:21:05.092594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519099704247466:2049] [node 2] ICS09 handshake done sender: [6:7571519095409280164:2120] self: [6:7571519095409280167:0] peer: [2:7571519096215246004:0] socket: 618 qp: -1
2025-11-11T17:21:05.092627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519099704247466:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:05.092655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:05.093047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519091114312779:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.093099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:05.093161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519091114312779:2050] [node 3] ICP22 created new session: [6:7571519099704247468:2050]
2025-11-11T17:21:05.093198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519099704247468:2050] [node 3] ICS09 handshake done sender: [6:7571519095409280158:2118] self: [6:7571519095409280161:0] peer: [3:7571519095199131397:0] socket: 601 qp: -1
2025-11-11T17:21:05.093227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519099704247468:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:05.093259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:05.102969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519099704247465:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:05.109067Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519099704247465:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:05.109212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519095409280125:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:05.126046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519099704247471:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24048
2025-11-11T17:21:05.142351Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519099704247465:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:05.143631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519095409280125:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:05.143699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:05.143764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519095409280125:2074] [node 4] ICP22 created new session: [6:7571519099704247472:2074]
2025-11-11T17:21:05.143810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519099704247472:2074] [node 4] ICS09 handshake done sender: [6:7571519099704247465:2123] self: [6:7571519099704247470:0] peer: [4:7571519093931998152:0] socket: 559 qp: -1
2025-11-11T17:21:05.143841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519099704247472:2074] [node 4] ICS10 traffic start
2025-11-11T17:21:05.143872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:05.125858Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519093931998153:2121] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:05.154163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519093931998130:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:05.154283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:05.154341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519093931998130:2108] [node 6] ICP22 created new session: [4:7571519098226965465:2108]
2025-11-11T17:21:05.154398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519098226965465:2108] [node 6] ICS09 handshake done sender: [4:7571519093931998153:2121] self: [4:7571519093931998152:0] peer: [6:7571519099704247470:0] socket: 594 qp: -1
2025-11-11T17:21:05.154444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519098226965465:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:05.154501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:05.267078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:05.267354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519100510213326:2048] [node 1] ICS01 socket: 604 reason# EndOfStream
2025-11-11T17:21:05.267396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519091920278633:2048] [node 1] ICP30 unregister session Session# [2:7571519100510213326:2048] VirtualId# [2:7571519096215246021:0]
2025-11-11T17:21:05.267432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519100510213326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.267549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:05.267597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:05.278511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:05.278782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519095199131410:2048] [node 1] ICS01 socket: 567 reason# EndOfStream
2025-11-11T17:21:05.278828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519090904164024:2048] [node 1] ICP30 unregister session Session# [3:7571519095199131410:2048] VirtualId# [3:7571519095199131404:0]
2025-11-11T17:21:05.278888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519095199131410:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.279015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:05.279062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:05.286421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:05.286616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519095409280154:2048] [node 1] ICS07 socket disconnect 565 reason# EndOfStream
2025-11-11T17:21:05.286649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519095409280154:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.286703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:05.286751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519095409280154:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:05.281780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:05.293764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:05.285989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519100587239659:2048] [node 1] ICS01 socket: 615 reason# EndOfStream
2025-11-11T17:21:05.286035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519091997304967:2048] [node 1] ICP30 unregister session Session# [5:7571519100587239659:2048] VirtualId# [5:7571519096292272359:0]
2025-11-11T17:21:05.286083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519100587239659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.286214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:05.286269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:05.294367Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address
2025-11-11T17:21:05.317726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519091114312777:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:05.320849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519095409280154:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:05.320913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519091114312777:2048] [node 1] ICP30 unregister session Session# [6:7571519095409280154:2048] VirtualId# [6:7571519095409280152:0]
2025-11-11T17:21:05.320981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:05.321070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519091114312777:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:05.321090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:05.321342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [6:7571519091114312777:2048] [node 1] ICP35 event processing took too much time StateWork:293(26547118ns) {HandleHandshakeStatus:395(26543293ns) {LogHandshakeFail:472(11640ns) {UpdateErrorStateLog:544(3275ns)} DropOutgoingHandshake:456(4977ns) ProcessPendingSessionEvents:495(627ns) UnregisterSession:538(59971ns) {ProcessPendingSessionEvents:495(456ns) SwitchToInitialState:201(8445ns) {SwitchToState:178(768ns)}} TransitToErrorState:752(111074ns) {SwitchToState:178(5871ns) ProcessPendingSessionEvents:495(110ns) DropHandshakes:469(3636ns) {DropIncomingHandshake:441(643ns) DropOutgoingHandshake:456(635ns)}}}}
2025-11-11T17:21:05.318135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519093931998161:2048] [node 1] ICS07 socket disconnect 572 reason# EndOfStream
2025-11-11T17:21:05.318188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519093931998161:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.318342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:05.318887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519093931998161:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:05.321816Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address
2025-11-11T17:21:05.345679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519089637030770:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:05.353716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519093931998161:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:05.353790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519089637030770:2048] [node 1] ICP30 unregister session Session# [4:7571519093931998161:2048] VirtualId# [4:7571519093931998147:0]
2025-11-11T17:21:05.353858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:05.353900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519089637030770:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:05.353918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:05.354239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [4:7571519089637030770:2048] [node 1] ICP35 event processing took too much time StateWork:293(31992096ns) {HandleHandshakeStatus:395(26918362ns) {LogHandshakeFail:472(18138ns) {UpdateErrorStateLog:544(4450ns)} DropOutgoingHandshake:456(8177ns) ProcessPendingSessionEvents:495(375ns) UnregisterSession:538(50778ns) {ProcessPendingSessionEvents:495(102ns) SwitchToInitialState:201(2482ns) {SwitchToState:178(1339ns)}} TransitToErrorState:752(61872ns) {SwitchToState:178(8387ns) ProcessPendingSessionEvents:495(50ns) DropHandshakes:469(2692ns) {DropIncomingHandshake:441(801ns) DropOutgoingHandshake:456(123ns)}}}}
2025-11-11T17:21:05.366127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519089637030770:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:05.350879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519091114312777:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:05.374793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:05.375053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519095199131417:2049] [node 2] ICS01 socket: 581 reason# EndOfStream
2025-11-11T17:21:05.375093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519090904164025:2049] [node 2] ICP30 unregister session Session# [3:7571519095199131417:2049] VirtualId# [3:7571519095199131412:0]
2025-11-11T17:21:05.375131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519095199131417:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.375233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:05.375282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:05.382384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:05.382580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519099704247466:2049] [node 2] ICS01 socket: 618 reason# EndOfStream
2025-11-11T17:21:05.382612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519091114312778:2049] [node 2] ICP30 unregister session Session# [6:7571519099704247466:2049] VirtualId# [6:7571519095409280167:0]
2025-11-11T17:21:05.382636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519099704247466:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.382688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:05.382721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:05.381817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:05.382072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519098226965462:2049] [node 2] ICS01 socket: 592 reason# EndOfStream
2025-11-11T17:21:05.382113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519089637030771:2049] [node 2] ICP30 unregister session Session# [4:7571519098226965462:2049] VirtualId# [4:7571519093931998151:0]
2025-11-11T17:21:05.382145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519098226965462:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.382230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:05.382277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:05.384620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:05.384849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519096292272356:2049] [node 2] ICS01 socket: 558 reason# EndOfStream
2025-11-11T17:21:05.384881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519091997304968:2049] [node 2] ICP30 unregister session Session# [5:7571519096292272356:2049] VirtualId# [5:7571519096292272346:0]
2025-11-11T17:21:05.384907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519096292272356:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.384989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:05.385039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:05.481836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:05.482773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:05.482069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519093931998156:2050] [node 3] ICS07 socket disconnect 578 reason# EndOfStream
2025-11-11T17:21:05.482973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519099704247468:2050] [node 3] ICS01 socket: 601 reason# EndOfStream
2025-11-11T17:21:05.482102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519093931998156:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.483010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519091114312779:2050] [node 3] ICP30 unregister session Session# [6:7571519099704247468:2050] VirtualId# [6:7571519095409280161:0]
2025-11-11T17:21:05.482188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:05.483050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519099704247468:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.482231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519093931998156:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:05.483111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:05.483156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:05.483684Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Couldn't connect to any resolved address
2025-11-11T17:21:05.483917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519089637030772:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:05.483963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519093931998156:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:05.483994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519089637030772:2050] [node 3] ICP30 unregister session Session# [4:7571519093931998156:2050] VirtualId# [4:7571519093931998148:0]
2025-11-11T17:21:05.484035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:05.484062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519089637030772:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:05.484075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:05.485786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:05.486055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519100587239657:2050] [node 3] ICS01 socket: 574 reason# EndOfStream
2025-11-11T17:21:05.486123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519091997304969:2050] [node 3] ICP30 unregister session Session# [5:7571519100587239657:2050] VirtualId# [5:7571519096292272345:0]
2025-11-11T17:21:05.486199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519100587239657:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.486370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:05.486464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:05.497765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519089637030772:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:05.593798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:05.594145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519100587239661:2100] [node 4] ICS01 socket: 587 reason# EndOfStream
2025-11-11T17:21:05.594221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519096292272319:2100] [node 4] ICP30 unregister session Session# [5:7571519100587239661:2100] VirtualId# [5:7571519096292272348:0]
2025-11-11T17:21:05.594280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519100587239661:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.594388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:05.594437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:05.597724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:05.598000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519099704247472:2074] [node 4] ICS01 socket: 559 reason# EndOfStream
2025-11-11T17:21:05.598049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519095409280125:2074] [node 4] ICP30 unregister session Session# [6:7571519099704247472:2074] VirtualId# [6:7571519099704247470:0]
2025-11-11T17:21:05.598084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519099704247472:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.598207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:05.598259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:05.690811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:05.691041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519095409280162:2072] [node 5] ICS01 socket: 591 reason# EndOfStream
2025-11-11T17:21:05.691087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519095409280099:2072] [node 5] ICP30 unregister session Session# [6:7571519095409280162:2072] VirtualId# [6:7571519095409280159:0]
2025-11-11T17:21:05.691122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519095409280162:2072] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:05.691224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:05.691274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:05.835493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519102109519160:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:05.835550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519102109519161:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:05.861308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519101420564500:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:05.861358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519101420564501:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:05.887581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519100242451369:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:05.887615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519100242451370:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:05.917461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519100006705100:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:05.917523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519100006705102:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:05.917662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519100006705101:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:05.946267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519098195134574:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:05.946311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519098195134575:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:05.946344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519098195134576:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:05.977087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519098069249952:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:05.977142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519098069249953:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:05.977220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519098069249954:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.229196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519104537418683:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51748
2025-11-11T17:21:06.286178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519104537418692:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.291579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217265:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:06.343258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519104301672430:2066] [node 5] ICP01 ready to work
2025-11-11T17:21:06.343776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519105715531850:2096] [node 6] ICP01 ready to work
2025-11-11T17:21:06.398454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519104301672439:2064] [node 6] ICP01 ready to work
2025-11-11T17:21:06.384753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519104537418728:2061] [node 4] ICP01 ready to work
2025-11-11T17:21:06.357208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519105715531849:2093] [node 5] ICP01 ready to work
2025-11-11T17:21:06.398087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519106404486519:2106] [node 5] ICP01 ready to work
2025-11-11T17:21:06.445714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519102490101894:2066] [node 4] ICP01 ready to work
2025-11-11T17:21:06.436103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519104537418735:2062] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.421702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519105715531859:2094] [node 4] ICP01 ready to work
2025-11-11T17:21:06.467323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519102490101905:2071] [node 6] ICP01 ready to work
2025-11-11T17:21:06.435715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519102364217266:2061] [node 4] ICP01 ready to work
2025-11-11T17:21:06.404197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519106404486520:2105] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.463532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519102109519161:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(244484776ns) {Configure:104(244472555ns) {ProcessConfigured:135(74162ns) {StartInitialHandshake:157(71519ns) {DropHandshakes:469(3167ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(552ns)} GenerateSessionVirtualId:988(1206ns) PrepareNewSessionHandshake:476(2102ns) {SwitchToState:178(802ns)}}}}}
2025-11-11T17:21:06.495788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519098069249953:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16348969ns) {Configure:104(16342193ns) {ProcessConfigured:135(91857ns) {StartInitialHandshake:157(88120ns) {DropHandshakes:469(4891ns) {DropIncomingHandshake:441(1392ns) DropOutgoingHandshake:456(683ns)} GenerateSessionVirtualId:988(1095ns) PrepareNewSessionHandshake:476(2257ns) {SwitchToState:178(989ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.558943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519100006705102:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(340788470ns) {Configure:104(340782568ns) {ProcessConfigured:135(102845ns) {StartInitialHandshake:157(100039ns) {DropHandshakes:469(7423ns) {DropIncomingHandshake:441(762ns) DropOutgoingHandshake:456(689ns)} GenerateSessionVirtualId:988(1104ns) PrepareNewSessionHandshake:476(2973ns) {SwitchToState:178(1050ns)}}}}}
2025-11-11T17:21:06.561300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519104301672463:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:06.538855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519098195134576:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(278599998ns) {Configure:104(278593605ns) {ProcessConfigured:135(80344ns) {StartInitialHandshake:157(76994ns) {DropHandshakes:469(3098ns) {DropIncomingHandshake:441(785ns) DropOutgoingHandshake:456(473ns)} GenerateSessionVirtualId:988(787ns) PrepareNewSessionHandshake:476(2056ns) {SwitchToState:178(635ns)}}}}}
2025-11-11T17:21:06.533491Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418683:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.497656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519106404486521:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21554
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.545901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519102364217265:2062] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.610661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519100006705101:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(290773042ns) {Configure:104(290762241ns) {ProcessConfigured:135(96899ns) {StartInitialHandshake:157(93874ns) {DropHandshakes:469(3717ns) {DropIncomingHandshake:441(876ns) DropOutgoingHandshake:456(765ns)} GenerateSessionVirtualId:988(1334ns) PrepareNewSessionHandshake:476(2510ns) {SwitchToState:178(700ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.539360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101946:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.574014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519102364217315:2067] [node 5] ICP01 ready to work
2025-11-11T17:21:06.513941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519106404486526:2107] [node 6] ICP01 ready to work
2025-11-11T17:21:06.662005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519104537418692:2063] [node 6] ICP35 event processing took too much time PendingConnection:278(55350041ns) {IncomingHandshake:261(11171369ns) {DropIncomingHandshake:441(1573ns) PrepareNewSessionHandshake:476(5267ns) {SwitchToState:178(3552ns)} GenerateSessionVirtualId:988(1032ns) IssueIncomingHandshakeReply:193(6549ns)}}
2025-11-11T17:21:06.662510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519104537418739:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51748
2025-11-11T17:21:06.662725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519104537418741:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51748
2025-11-11T17:21:06.665805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519100242451369:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(356484922ns) {Configure:104(356479530ns) {ProcessConfigured:135(23828354ns) {StartInitialHandshake:157(23824274ns) {DropHandshakes:469(3519ns) {DropIncomingHandshake:441(683ns) DropOutgoingHandshake:456(601ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:476(5824ns) {SwitchToState:178(3294ns)}}}}}
2025-11-11T17:21:06.675652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519104537418735:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21511906ns) {Configure:104(21504437ns) {ProcessConfigured:135(73772ns) {StartInitialHandshake:157(70594ns) {DropHandshakes:469(3269ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(668ns)} GenerateSessionVirtualId:988(1323ns) PrepareNewSessionHandshake:476(2310ns) {SwitchToState:178(852ns)}}}}}
2025-11-11T17:21:06.627391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519104301672470:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:06.629423Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519104301672463:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.649663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519104301672470:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.633928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101947:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.647996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519098069249954:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.569825Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486521:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.703333Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418739:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.726894Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418694:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.726953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519104537418694:2074] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.727747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519100242451369:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.728474Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519104537418694:2074] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.655092Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519102490101946:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.721982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519104301672465:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.594843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519106404486520:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(129401358ns) {Configure:104(129396184ns) {ProcessConfigured:135(90264ns) {StartInitialHandshake:157(84139ns) {DropHandshakes:469(3977ns) {DropIncomingHandshake:441(1032ns) DropOutgoingHandshake:456(669ns)} GenerateSessionVirtualId:988(1943ns) PrepareNewSessionHandshake:476(2070ns) {SwitchToState:178(819ns)}}}}}
2025-11-11T17:21:06.669757Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519098069249954:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18178293ns) {IncomingHandshake:261(18175731ns) {DropIncomingHandshake:441(621ns) PrepareNewSessionHandshake:476(2609ns) {SwitchToState:178(1159ns)} GenerateSessionVirtualId:988(1266ns) IssueIncomingHandshakeReply:193(7860301ns) {SwitchToState:178(3927ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.728822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519104537418692:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.655316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519098195134574:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.625973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519106404486526:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(103781329ns) {Configure:104(103774104ns) {ProcessConfigured:135(56026602ns) {StartInitialHandshake:157(56022084ns) {DropHandshakes:469(3907ns) {DropIncomingHandshake:441(758ns) DropOutgoingHandshake:456(502ns)} GenerateSessionVirtualId:988(1395ns) PrepareNewSessionHandshake:476(6033ns) {SwitchToState:178(3136ns)}}}}}
2025-11-11T17:21:06.766201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519101420564501:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(391017050ns) {Configure:104(391012193ns) {ProcessConfigured:135(93534ns) {StartInitialHandshake:157(89761ns) {DropHandshakes:469(4228ns) {DropIncomingHandshake:441(1103ns) DropOutgoingHandshake:456(533ns)} GenerateSessionVirtualId:988(663ns) PrepareNewSessionHandshake:476(2501ns) {SwitchToState:178(842ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.734493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519102364217266:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(199793695ns) {Configure:104(199786131ns) {ProcessConfigured:135(18495836ns) {StartInitialHandshake:157(18493085ns) {DropHandshakes:469(4030ns) {DropIncomingHandshake:441(747ns) DropOutgoingHandshake:456(805ns)} GenerateSessionVirtualId:988(1684ns) PrepareNewSessionHandshake:476(2926ns) {SwitchToState:178(1544ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.728931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:06.722050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519104301672465:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.681874Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519102490101947:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.638267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519102109519160:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(424622373ns) {Configure:104(424617463ns) {ProcessConfigured:135(40211680ns) {StartInitialHandshake:157(40207854ns) {DropHandshakes:469(3670ns) {DropIncomingHandshake:441(862ns) DropOutgoingHandshake:456(651ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:476(5078ns) {SwitchToState:178(2811ns)}}}}}
2025-11-11T17:21:06.759922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217323:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:06.768752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519105715531860:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17694
2025-11-11T17:21:06.728984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519104537418692:2063] [node 6] ICP22 created new session: [3:7571519104537418749:2063]
2025-11-11T17:21:06.785986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519105715531849:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(262208231ns) {Configure:104(262202391ns) {ProcessConfigured:135(15790683ns) {StartInitialHandshake:157(15787728ns) {DropHandshakes:469(2557ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(471ns)} GenerateSessionVirtualId:988(631ns) PrepareNewSessionHandshake:476(5960ns) {SwitchToState:178(2981ns)}}}}}
2025-11-11T17:21:06.786145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519105715531863:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17694
2025-11-11T17:21:06.786642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519105715531869:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17694
2025-11-11T17:21:06.814194Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531869:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.833787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519105715531862:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17694
2025-11-11T17:21:06.854001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519105715531850:2096] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(449306058ns) {Configure:104(449297217ns) {ProcessConfigured:135(39587419ns) {StartInitialHandshake:157(39583809ns) {DropHandshakes:469(2497ns) {DropIncomingHandshake:441(886ns) DropOutgoingHandshake:456(229ns)} GenerateSessionVirtualId:988(1133ns) PrepareNewSessionHandshake:476(4622ns) {SwitchToState:178(1702ns)}}}}}
2025-11-11T17:21:06.856196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519101420564500:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(432065547ns) {Configure:104(432056364ns) {ProcessConfigured:135(16956546ns) {StartInitialHandshake:157(16952396ns) {DropHandshakes:469(3557ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(716ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:476(5904ns) {SwitchToState:178(2910ns)}}}}}
2025-11-11T17:21:06.870520Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531863:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.760526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519100006705100:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(416902486ns) {Configure:104(416895950ns) {ProcessConfigured:135(25885619ns) {StartInitialHandshake:157(17568609ns) {DropHandshakes:469(4058ns) {DropIncomingHandshake:441(645ns) DropOutgoingHandshake:456(810ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:476(5073ns) {SwitchToState:178(2491ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.883302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519105715531879:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17694
2025-11-11T17:21:06.682094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519102490101894:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.686306Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486525:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.729026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519104537418749:2063] [node 6] ICS09 handshake done sender: [3:7571519104537418694:2074] self: [3:7571519104537418693:0] peer: [6:7571519102364217320:0] socket: 567 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.760722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519100006705100:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.714145Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:06.686349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519106404486525:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.729066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519104537418749:2063] [node 6] ICS10 traffic start
2025-11-11T17:21:06.886040Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531862:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.760122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217324:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.715078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519098195134574:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519102490101946:2114] held: no
2025-11-11T17:21:06.763231Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519104301672465:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:06.769730Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9418) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:06.729103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:06.741268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519104537418751:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51748
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.886181Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531860:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.710709Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519106404486525:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.890835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531871:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.750120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519102490101905:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(282457043ns) {Configure:104(282451813ns) {ProcessConfigured:135(93329ns) {StartInitialHandshake:157(90393ns) {DropHandshakes:469(7816ns) {DropIncomingHandshake:441(635ns) DropOutgoingHandshake:456(688ns)} GenerateSessionVirtualId:988(1422ns) PrepareNewSessionHandshake:476(2763ns) {SwitchToState:178(1029ns)}}}}}
2025-11-11T17:21:06.769137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519104301672439:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(310367189ns) {Configure:104(310356712ns) {ProcessConfigured:135(13368319ns) {StartInitialHandshake:157(13365868ns) {DropHandshakes:469(4241ns) {DropIncomingHandshake:441(1213ns) DropOutgoingHandshake:456(752ns)} GenerateSessionVirtualId:988(931ns) PrepareNewSessionHandshake:476(2758ns) {SwitchToState:178(1244ns)}}}}}
2025-11-11T17:21:06.763512Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418741:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.775313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519102364217315:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(134958210ns) {Configure:104(134949510ns) {ProcessConfigured:135(102958ns) {StartInitialHandshake:157(100391ns) {DropHandshakes:469(3475ns) {DropIncomingHandshake:441(702ns) DropOutgoingHandshake:456(729ns)} GenerateSessionVirtualId:988(1212ns) PrepareNewSessionHandshake:476(2902ns) {SwitchToState:178(1270ns)}}}}}
2025-11-11T17:21:06.711040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519106404486519:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.711122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:06.711186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519106404486519:2106] [node 5] ICP22 created new session: [1:7571519106404486539:2106]
2025-11-11T17:21:06.711237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519106404486539:2106] [node 5] ICS09 handshake done sender: [1:7571519106404486525:2112] self: [1:7571519106404486524:0] peer: [5:7571519102490101948:0] socket: 578 qp: -1
2025-11-11T17:21:06.711277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519106404486539:2106] [node 5] ICS10 traffic start
2025-11-11T17:21:06.711329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:06.763279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519106404486537:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21554
2025-11-11T17:21:06.765839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486532:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.765874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519106404486532:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.789782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486523:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.789820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519106404486523:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.813881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519106404486532:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:06.838366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519106404486520:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.838436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:06.838488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519106404486520:2105] [node 4] ICP22 created new session: [1:7571519106404486544:2105]
2025-11-11T17:21:06.838535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519106404486544:2105] [node 4] ICS09 handshake done sender: [1:7571519106404486532:2113] self: [1:7571519106404486531:0] peer: [4:7571519104301672477:0] socket: 586 qp: -1
2025-11-11T17:21:06.838596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519106404486544:2105] [node 4] ICS10 traffic start
2025-11-11T17:21:06.838660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:06.839663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519106404486543:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21554
2025-11-11T17:21:06.840203Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519106404486523:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:06.840498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519102109519161:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.840557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:06.840604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519102109519161:2049] [node 3] ICP22 created new session: [1:7571519106404486548:2049]
2025-11-11T17:21:06.840654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519106404486548:2049] [node 3] ICS09 handshake done sender: [1:7571519106404486523:2111] self: [1:7571519106404486522:0] peer: [3:7571519104537418748:0] socket: 575 qp: -1
2025-11-11T17:21:06.840685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519106404486548:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:06.840737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:06.846003Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486537:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.858599Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486543:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.858780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519106404486520:2105] [node 4] ICP18 handshake (actor [1:7571519106404486543:2117]) is too old
2025-11-11T17:21:06.893778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486535:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.893822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519106404486535:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.894490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519102109519161:2049] [node 3] ICP18 handshake (actor [1:7571519106404486537:2116]) is too old
2025-11-11T17:21:06.925820Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519106404486535:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:06.930619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519106404486526:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.930671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:06.930719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519106404486526:2107] [node 6] ICP22 created new session: [1:7571519106404486552:2107]
2025-11-11T17:21:06.930787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519106404486552:2107] [node 6] ICS09 handshake done sender: [1:7571519106404486535:2114] self: [1:7571519106404486529:0] peer: [6:7571519102364217332:0] socket: 592 qp: -1
2025-11-11T17:21:06.930835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519106404486552:2107] [node 6] ICS10 traffic start
2025-11-11T17:21:06.930874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:06.945602Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486536:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.945655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519106404486536:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.962323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519106404486551:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21554
2025-11-11T17:21:06.978542Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519106404486536:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:06.985835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519106404486554:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21554
2025-11-11T17:21:06.985903Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:06.989750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519106404486551:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.777915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101953:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.890860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519105715531871:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:07.011855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519102109519160:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:07.012018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:07.012074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519102109519160:2048] [node 2] ICP22 created new session: [1:7571519110699453853:2048]
2025-11-11T17:21:07.012126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519110699453853:2048] [node 2] ICS09 handshake done sender: [1:7571519106404486536:2115] self: [1:7571519106404486534:0] peer: [2:7571519105715531884:0] socket: 613 qp: -1
2025-11-11T17:21:07.012164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519110699453853:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:07.012213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:06.794330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519104537418753:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51748
2025-11-11T17:21:06.775521Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519102364217323:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.794874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519104301672430:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.787997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101954:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.897910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531879:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.775631Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519102364217265:2062] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:06.794980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:06.817673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418751:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.898278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519101420564500:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.901479Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519105715531871:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:06.925652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531868:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.925697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519105715531868:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.933993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519105715531849:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.940141Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519105715531868:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:06.940448Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531866:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.940482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519105715531866:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.958125Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519105715531866:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:06.978195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:06.982209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519105715531859:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.982318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:06.982362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519105715531859:2094] [node 4] ICP22 created new session: [2:7571519105715531890:2094]
2025-11-11T17:21:06.982406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519105715531890:2094] [node 4] ICS09 handshake done sender: [2:7571519105715531868:2114] self: [2:7571519105715531867:0] peer: [4:7571519104301672489:0] socket: 604 qp: -1
2025-11-11T17:21:06.982447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519105715531890:2094] [node 4] ICS10 traffic start
2025-11-11T17:21:06.982497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:06.983529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519101420564501:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.983636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:06.983681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519101420564501:2049] [node 3] ICP22 created new session: [2:7571519105715531892:2049]
2025-11-11T17:21:06.983732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519105715531892:2049] [node 3] ICS09 handshake done sender: [2:7571519105715531866:2113] self: [2:7571519105715531865:0] peer: [3:7571519104537418760:0] socket: 603 qp: -1
2025-11-11T17:21:06.983759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519105715531892:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:06.983791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:06.989882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519105715531849:2093] [node 5] ICP22 created new session: [2:7571519105715531889:2093]
2025-11-11T17:21:07.005682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519105715531889:2093] [node 5] ICS09 handshake done sender: [2:7571519105715531871:2116] self: [2:7571519105715531870:0] peer: [5:7571519102490101965:0] socket: 605 qp: -1
2025-11-11T17:21:07.005741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519105715531889:2093] [node 5] ICS10 traffic start
2025-11-11T17:21:07.006195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519105715531895:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17694
2025-11-11T17:21:07.021986Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519105715531879:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:07.034289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:07.060031Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:07.066005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519101420564500:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:07.066097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:07.066161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519101420564500:2048] [node 1] ICP22 created new session: [2:7571519110010499193:2048]
2025-11-11T17:21:07.066208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519110010499193:2048] [node 1] ICS09 handshake done sender: [2:7571519105715531879:2119] self: [2:7571519105715531884:0] peer: [1:7571519106404486534:0] socket: 616 qp: -1
2025-11-11T17:21:07.066245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519110010499193:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:07.066283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:07.066567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519101420564500:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:06.870675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519104537418728:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(410276014ns) {Configure:104(410267998ns) {ProcessConfigured:135(19701025ns) {StartInitialHandshake:157(19698360ns) {DropHandshakes:469(3216ns) {DropIncomingHandshake:441(784ns) DropOutgoingHandshake:456(687ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:476(6134ns) {SwitchToState:178(3179ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.795898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519104301672430:2066] [node 5] ICP22 created new session: [4:7571519104301672479:2066]
2025-11-11T17:21:06.789665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101955:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.802061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519098069249954:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9418) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519102364217265:2062] held: no
2025-11-11T17:21:06.806427Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31038) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:06.807427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519102490101894:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31038) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519102490101947:2115] held: no
2025-11-11T17:21:06.807577Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519102490101946:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:06.807836Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519102490101947:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:06.807991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519098195134574:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.808040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:06.808083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519098195134574:2048] [node 1] ICP22 created new session: [5:7571519102490101957:2048]
2025-11-11T17:21:06.808127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519102490101957:2048] [node 1] ICS09 handshake done sender: [5:7571519102490101946:2114] self: [5:7571519102490101948:0] peer: [1:7571519106404486524:0] socket: 579 qp: -1
2025-11-11T17:21:06.808161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519102490101957:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:06.808192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:06.808619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101956:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.808654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519102490101894:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.808685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:06.808733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519102490101894:2066] [node 4] ICP22 created new session: [5:7571519102490101959:2066]
2025-11-11T17:21:06.808775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519102490101959:2066] [node 4] ICS09 handshake done sender: [5:7571519102490101947:2115] self: [5:7571519102490101949:0] peer: [4:7571519104301672464:0] socket: 585 qp: -1
2025-11-11T17:21:06.808802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519102490101959:2066] [node 4] ICS10 traffic start
2025-11-11T17:21:06.808830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:06.821842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519102490101954:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.823218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519098195134576:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.890424Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519102490101956:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.890563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519098195134575:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.895588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101964:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.899051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101966:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.899457Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519102490101954:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:06.899686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519098195134576:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.899764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:06.899815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519098195134576:2050] [node 3] ICP22 created new session: [5:7571519102490101967:2050]
2025-11-11T17:21:06.899866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519102490101967:2050] [node 3] ICS09 handshake done sender: [5:7571519102490101954:2118] self: [5:7571519102490101962:0] peer: [3:7571519104537418745:0] socket: 565 qp: -1
2025-11-11T17:21:06.899901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519102490101967:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:06.899936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:06.906325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519102490101952:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.906366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519102490101952:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.910037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519102490101963:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19236
2025-11-11T17:21:06.918345Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519102490101963:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.918445Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519102490101956:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:06.920735Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519102490101952:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:06.920972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519102490101905:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.921060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:06.921107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519102490101905:2071] [node 6] ICP22 created new session: [5:7571519102490101971:2071]
2025-11-11T17:21:06.921164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519102490101971:2071] [node 6] ICS09 handshake done sender: [5:7571519102490101952:2116] self: [5:7571519102490101951:0] peer: [6:7571519102364217333:0] socket: 572 qp: -1
2025-11-11T17:21:06.921200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519102490101971:2071] [node 6] ICS10 traffic start
2025-11-11T17:21:06.921242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:06.923510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519098195134575:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.923571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:06.923624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519098195134575:2049] [node 2] ICP22 created new session: [5:7571519102490101973:2049]
2025-11-11T17:21:06.923680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519102490101973:2049] [node 2] ICS09 handshake done sender: [5:7571519102490101956:2120] self: [5:7571519102490101965:0] peer: [2:7571519105715531870:0] socket: 611 qp: -1
2025-11-11T17:21:06.926790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519102490101973:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:06.926849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:07.082248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519105715531849:2093] [node 5] ICP35 event processing took too much time PendingConnection:278(148062209ns) {HandleHandshakeStatus:329(148057963ns) {DropIncomingHandshake:441(29410ns) DropOutgoingHandshake:456(1051ns) SwitchToState:178(909ns) ProcessPendingSessionEvents:495(133328ns) {ForwardSessionEventToSession:612(80356ns) ForwardSessionEventToSession:612(7354ns) ForwardSessionEventToSession:612(5297ns) ForwardSessionEventToSession:612(2968ns) ForwardSessionEventToSession:612(8499ns) ForwardSessionEventToSession:612(3354ns) ForwardSessionEventToSession:612(3124ns) ForwardSessionEventToSession:612(5311ns) ForwardSessionEventToSession:612(2393ns) ForwardSessionEventToSession:612(4875ns)}}}
2025-11-11T17:21:06.795961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519104301672479:2066] [node 5] ICS09 handshake done sender: [4:7571519104301672465:2112] self: [4:7571519104301672464:0] peer: [5:7571519102490101949:0] socket: 584 qp: -1
2025-11-11T17:21:06.842131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217330:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:06.870895Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418746:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:06.857875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519098069249952:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(391974986ns) {Configure:104(391968956ns) {ProcessConfigured:135(63993930ns) {StartInitialHandshake:157(56107601ns) {DropHandshakes:469(5644ns) {DropIncomingHandshake:441(1326ns) DropOutgoingHandshake:456(792ns)} GenerateSessionVirtualId:988(3432ns) PrepareNewSessionHandshake:476(3609ns) {SwitchToState:178(1452ns)}}}}}
2025-11-11T17:21:06.858006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519098069249952:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.883379Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519102364217330:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.883557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519102364217315:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.902514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217334:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:06.913980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217335:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:06.937942Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9291) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:06.939691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519102364217315:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9291) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519102364217330:2118] held: no
2025-11-11T17:21:06.940054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217336:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:06.945895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519098069249954:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(85542029ns) {HandleHandshakeStatus:395(85539873ns) {LogHandshakeFail:472(7933254ns) {UpdateErrorStateLog:544(12652ns)} DropOutgoingHandshake:456(8630ns) ProcessPendingSessionEvents:495(45453717ns) {DropSessionEvent:512(31927014ns) DropSessionEvent:512(19077ns) DropSessionEvent:512(4750ns) DropSessionEvent:512(7049ns) DropSessionEvent:512(10405681ns) DropSessionEvent:512(6709ns) DropSessionEvent:512(6537ns) DropSessionEvent:512(3049698ns) DropSessionEvent:512(6281ns) DropSessionEvent:512(5350ns)}}}
2025-11-11T17:21:06.946284Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519102364217323:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:06.946727Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519102364217330:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:06.947152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519098069249952:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.947210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:06.947264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519098069249952:2048] [node 1] ICP22 created new session: [6:7571519102364217338:2048]
2025-11-11T17:21:06.947309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519102364217338:2048] [node 1] ICS09 handshake done sender: [6:7571519102364217323:2114] self: [6:7571519102364217332:0] peer: [1:7571519106404486529:0] socket: 595 qp: -1
2025-11-11T17:21:06.947345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519102364217338:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:06.947382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:06.948585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519102364217315:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.948627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:06.948676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519102364217315:2067] [node 5] ICP22 created new session: [6:7571519102364217340:2067]
2025-11-11T17:21:06.948726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519102364217340:2067] [node 5] ICS09 handshake done sender: [6:7571519102364217330:2118] self: [6:7571519102364217333:0] peer: [5:7571519102490101951:0] socket: 614 qp: -1
2025-11-11T17:21:06.948758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519102364217340:2067] [node 5] ICS10 traffic start
2025-11-11T17:21:06.948789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:06.961995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519098069249954:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.991401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:06.991594Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519102364217334:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.997634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519098069249954:2050] [node 3] ICP22 created new session: [6:7571519102364217342:2050]
2025-11-11T17:21:06.999952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519102364217343:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:07.009935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519102364217266:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:07.013733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519102364217342:2050] [node 3] ICS09 handshake done sender: [6:7571519102364217265:2062] self: [6:7571519102364217320:0] peer: [3:7571519104537418693:0] socket: 569 qp: -1
2025-11-11T17:21:07.022578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519102364217342:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:07.046199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:07.077973Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519102364217343:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:07.078152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519098069249953:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:07.114373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519106659184644:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:07.114818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519106659184643:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-11T17:21:07.115047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519098069249954:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(148428692ns) {HandleHandshakeStatus:329(148425451ns) {DropIncomingHandshake:441(1724ns) DropOutgoingHandshake:456(530ns) SwitchToState:178(753ns) ProcessPendingSessionEvents:495(1337ns)}}
2025-11-11T17:21:06.870941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519104537418746:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:06.878115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519104537418757:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51748
2025-11-11T17:21:06.895058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418753:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.895191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519100242451370:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.906120Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519104537418746:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:06.906478Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519104537418739:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:06.950499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519104537418735:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:06.950559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:06.950596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519104537418735:2062] [node 5] ICP22 created new session: [3:7571519104537418761:2062]
2025-11-11T17:21:06.950641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519104537418761:2062] [node 5] ICS09 handshake done sender: [3:7571519104537418746:2117] self: [3:7571519104537418745:0] peer: [5:7571519102490101962:0] socket: 596 qp: -1
2025-11-11T17:21:06.950673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519104537418761:2062] [node 5] ICS10 traffic start
2025-11-11T17:21:06.950702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:06.951406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519100242451369:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.959207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519104537418763:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51748
2025-11-11T17:21:06.959724Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519104537418753:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:06.959977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519100242451370:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.960031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:06.960086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519100242451370:2049] [node 2] ICP22 created new session: [3:7571519104537418765:2049]
2025-11-11T17:21:06.960137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519104537418765:2049] [node 2] ICS09 handshake done sender: [3:7571519104537418753:2119] self: [3:7571519104537418760:0] peer: [2:7571519105715531865:0] socket: 606 qp: -1
2025-11-11T17:21:06.960173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519104537418765:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:06.960208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:06.971037Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:06.971655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:06.978146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519100242451369:2048] [node 1] ICP22 created new session: [3:7571519104537418767:2048]
2025-11-11T17:21:06.985745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519104537418767:2048] [node 1] ICS09 handshake done sender: [3:7571519104537418739:2114] self: [3:7571519104537418748:0] peer: [1:7571519106404486522:0] socket: 580 qp: -1
2025-11-11T17:21:06.991769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519104537418767:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:06.991820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:06.998404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519100242451369:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(56433504ns) {HandleHandshakeStatus:329(56429256ns) {DropIncomingHandshake:441(2776ns) DropOutgoingHandshake:456(5895ns) SwitchToState:178(236ns) ProcessPendingSessionEvents:495(120253ns) {ForwardSessionEventToSession:612(71807ns) ForwardSessionEventToSession:612(3662ns) ForwardSessionEventToSession:612(8598ns) ForwardSessionEventToSession:612(2549ns) ForwardSessionEventToSession:612(4933ns) ForwardSessionEventToSession:612(5735ns) ForwardSessionEventToSession:612(3017ns) ForwardSessionEventToSession:612(5367ns) ForwardSessionEventToSession:612(2737ns) ForwardSessionEventToSession:612(2724ns)}}}
2025-11-11T17:21:06.998579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519100242451369:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:07.082432Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519105715531878:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.796000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519104301672479:2066] [node 5] ICS10 traffic start
2025-11-11T17:21:06.796052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:06.798473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519104301672482:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:06.798592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519104301672483:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:06.826414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519104301672484:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:06.837981Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519104301672482:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.850822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519104301672470:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:06.862055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519100006705100:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:06.862124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:06.862173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519100006705100:2048] [node 1] ICP22 created new session: [4:7571519104301672486:2048]
2025-11-11T17:21:06.862222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519104301672486:2048] [node 1] ICS09 handshake done sender: [4:7571519104301672470:2114] self: [4:7571519104301672477:0] peer: [1:7571519106404486531:0] socket: 587 qp: -1
2025-11-11T17:21:06.862267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519104301672486:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:06.862317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:06.881912Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:06.885183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519100006705100:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:06.885338Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519104301672483:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:06.885541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519100006705101:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:06.990489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519104301672490:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:07.025886Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7749) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:07.028792Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519104301672483:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:07.029037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519100006705101:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7749) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519104301672483:2118] held: no
2025-11-11T17:21:07.029134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519100006705101:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:07.126125Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31038) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:07.133683Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519102364217343:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:07.134472Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519104537418755:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:07.134516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519104537418755:2120] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:07.135966Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519104537418755:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:07.082456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519105715531878:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:07.138015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519102364217266:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31038) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519102364217334:2119] held: no
2025-11-11T17:21:07.029235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:07.137283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519104537418728:2061] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:07.157850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519098069249953:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:07.086084Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519105715531878:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:07.129802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519105715531850:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:07.157978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:07.029293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519100006705101:2049] [node 2] ICP22 created new session: [4:7571519108596639788:2049]
2025-11-11T17:21:07.137356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:07.137404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519104537418728:2061] [node 4] ICP22 created new session: [3:7571519108832386066:2061]
2025-11-11T17:21:07.137454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519108832386066:2061] [node 4] ICS09 handshake done sender: [3:7571519104537418755:2120] self: [3:7571519104537418754:0] peer: [4:7571519108596639791:0] socket: 570 qp: -1
2025-11-11T17:21:07.137488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519108832386066:2061] [node 4] ICS10 traffic start
2025-11-11T17:21:07.137527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:07.162044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:07.170126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519105715531850:2096] [node 6] ICP22 created new session: [2:7571519110010499196:2096]
2025-11-11T17:21:07.174727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519110010499196:2096] [node 6] ICS09 handshake done sender: [2:7571519105715531878:2118] self: [2:7571519105715531877:0] peer: [6:7571519106659184642:0] socket: 589 qp: -1
2025-11-11T17:21:07.174784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519110010499196:2096] [node 6] ICS10 traffic start
2025-11-11T17:21:07.174843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:07.158044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519098069249953:2049] [node 2] ICP22 created new session: [6:7571519106659184645:2049]
2025-11-11T17:21:07.165693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519106659184645:2049] [node 2] ICS09 handshake done sender: [6:7571519102364217343:2122] self: [6:7571519106659184642:0] peer: [2:7571519105715531877:0] socket: 628 qp: -1
2025-11-11T17:21:07.165763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519106659184645:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:07.165848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:07.166475Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7749) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:07.167884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519098069249953:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:07.029347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519108596639788:2049] [node 2] ICS09 handshake done sender: [4:7571519104301672483:2118] self: [4:7571519104301672489:0] peer: [2:7571519105715531867:0] socket: 608 qp: -1
2025-11-11T17:21:07.029385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519108596639788:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:07.029423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:07.053807Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519104301672478:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:07.053862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519104301672478:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:07.093712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519104301672491:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:07.097956Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519104301672478:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:07.118067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519104301672439:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:07.133778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519104301672491:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:07.133997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519100006705102:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:07.135789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519108596639792:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15993
2025-11-11T17:21:07.136997Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519104301672491:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:07.137799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519100006705102:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:07.152076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:07.152138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519104301672439:2064] [node 6] ICP22 created new session: [4:7571519108596639793:2064]
2025-11-11T17:21:07.158409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:07.158467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519100006705102:2050] [node 3] ICP22 created new session: [4:7571519108596639794:2050]
2025-11-11T17:21:07.158540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519108596639794:2050] [node 3] ICS09 handshake done sender: [4:7571519104301672491:2121] self: [4:7571519108596639791:0] peer: [3:7571519104537418754:0] socket: 568 qp: -1
2025-11-11T17:21:07.158597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519108596639794:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:07.158662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:07.159140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519100006705102:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(21108208ns) {HandleHandshakeStatus:329(21106153ns) {DropIncomingHandshake:441(281ns) DropOutgoingHandshake:456(6669ns) SwitchToState:178(316ns) ProcessPendingSessionEvents:495(103406ns) {ForwardSessionEventToSession:612(49912ns) ForwardSessionEventToSession:612(6641ns) ForwardSessionEventToSession:612(3094ns) ForwardSessionEventToSession:612(5668ns) ForwardSessionEventToSession:612(4964ns) ForwardSessionEventToSession:612(5553ns) ForwardSessionEventToSession:612(4208ns) ForwardSessionEventToSession:612(5870ns) ForwardSessionEventToSession:612(2899ns) ForwardSessionEventToSession:612(5190ns)}}}
2025-11-11T17:21:07.169759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519108596639793:2064] [node 6] ICS09 handshake done sender: [4:7571519104301672478:2116] self: [4:7571519104301672475:0] peer: [6:7571519106659184640:0] socket: 581 qp: -1
2025-11-11T17:21:07.169832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519108596639793:2064] [node 6] ICS10 traffic start
2025-11-11T17:21:07.169918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:07.174348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519104301672439:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(56207957ns) {HandleHandshakeStatus:329(56203443ns) {DropIncomingHandshake:441(16638ns) DropOutgoingHandshake:456(1038ns) SwitchToState:178(1074ns) ProcessPendingSessionEvents:495(4009474ns) {ForwardSessionEventToSession:612(3952678ns) ForwardSessionEventToSession:612(11838ns) ForwardSessionEventToSession:612(2955ns) ForwardSessionEventToSession:612(3025ns) ForwardSessionEventToSession:612(5470ns) ForwardSessionEventToSession:612(3273ns) ForwardSessionEventToSession:612(6056ns) ForwardSessionEventToSession:612(2810ns) ForwardSessionEventToSession:612(3131ns) ForwardSessionEventToSession:612(7327ns)}}}
2025-11-11T17:21:07.180606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519102364217266:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(25200955ns) {HandleHandshakeStatus:395(25196236ns) {LogHandshakeFail:472(21087ns) {UpdateErrorStateLog:544(5381ns)} DropOutgoingHandshake:456(2175ns) ProcessPendingSessionEvents:495(13954019ns) {DropSessionEvent:512(21505ns) DropSessionEvent:512(3895968ns) DropSessionEvent:512(6469ns) DropSessionEvent:512(5359ns) DropSessionEvent:512(5065ns) DropSessionEvent:512(6379ns) DropSessionEvent:512(6183ns) DropSessionEvent:512(5489ns) DropSessionEvent:512(5403ns) DropSessionEvent:512(9986644ns)}}}
2025-11-11T17:21:07.180738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519102364217334:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:07.178616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519105715531850:2096] [node 6] ICP35 event processing took too much time PendingConnection:278(48636189ns) {HandleHandshakeStatus:329(48632008ns) {DropIncomingHandshake:441(30895ns) DropOutgoingHandshake:456(767ns) SwitchToState:178(634ns) ProcessPendingSessionEvents:495(3435716ns) {ForwardSessionEventToSession:612(3369992ns) ForwardSessionEventToSession:612(8842ns) ForwardSessionEventToSession:612(16280ns) ForwardSessionEventToSession:612(3235ns) ForwardSessionEventToSession:612(5364ns) ForwardSessionEventToSession:612(3398ns) ForwardSessionEventToSession:612(6025ns) ForwardSessionEventToSession:612(3005ns) ForwardSessionEventToSession:612(3324ns) ForwardSessionEventToSession:612(4952ns)}}}
2025-11-11T17:21:07.180891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519104537418728:2061] [node 4] ICP35 event processing took too much time PendingConnection:278(24869357ns) {HandleHandshakeStatus:329(24866549ns) {DropIncomingHandshake:441(14467ns) DropOutgoingHandshake:456(548ns) SwitchToState:178(651ns) ProcessPendingSessionEvents:495(24480607ns) {ForwardSessionEventToSession:612(9028038ns) ForwardSessionEventToSession:612(5631ns) ForwardSessionEventToSession:612(5270455ns) ForwardSessionEventToSession:612(6180660ns) ForwardSessionEventToSession:612(10247ns) ForwardSessionEventToSession:612(3501548ns) ForwardSessionEventToSession:612(4785ns) ForwardSessionEventToSession:612(3273ns) ForwardSessionEventToSession:612(447186ns) ForwardSessionEventToSession:612(3348ns)}}}
2025-11-11T17:21:07.189950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519102364217266:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:07.190081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:07.190140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519102364217266:2061] [node 4] ICP22 created new session: [6:7571519106659184647:2061]
2025-11-11T17:21:07.190191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519106659184647:2061] [node 4] ICS09 handshake done sender: [6:7571519102364217334:2119] self: [6:7571519106659184640:0] peer: [4:7571519104301672475:0] socket: 619 qp: -1
2025-11-11T17:21:07.190235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519106659184647:2061] [node 4] ICS10 traffic start
2025-11-11T17:21:07.190287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:07.289779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:07.290040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519110010499193:2048] [node 1] ICS01 socket: 616 reason# EndOfStream
2025-11-11T17:21:07.290087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519101420564500:2048] [node 1] ICP30 unregister session Session# [2:7571519110010499193:2048] VirtualId# [2:7571519105715531884:0]
2025-11-11T17:21:07.290131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519110010499193:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.290231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:07.290277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:07.297748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:07.298007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519104537418767:2048] [node 1] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-11T17:21:07.298043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519104537418767:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.298164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:07.298213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519104537418767:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:07.309839Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address
2025-11-11T17:21:07.310248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519100242451369:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:07.310308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519104537418767:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:07.310347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519100242451369:2048] [node 1] ICP30 unregister session Session# [3:7571519104537418767:2048] VirtualId# [3:7571519104537418748:0]
2025-11-11T17:21:07.310408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:07.310448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519100242451369:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:07.310463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:07.322937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519100242451369:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:07.321854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:07.322134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519102364217338:2048] [node 1] ICS07 socket disconnect 595 reason# EndOfStream
2025-11-11T17:21:07.322857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:07.322168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519102364217338:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.322279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:07.323118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519102490101957:2048] [node 1] ICS01 socket: 579 reason# EndOfStream
2025-11-11T17:21:07.322320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519102364217338:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:07.323158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519098195134574:2048] [node 1] ICP30 unregister session Session# [5:7571519102490101957:2048] VirtualId# [5:7571519102490101948:0]
2025-11-11T17:21:07.323194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519102490101957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.323224Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address
2025-11-11T17:21:07.323308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:07.323845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519098069249952:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:07.323357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:07.323906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519102364217338:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:07.323949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519098069249952:2048] [node 1] ICP30 unregister session Session# [6:7571519102364217338:2048] VirtualId# [6:7571519102364217332:0]
2025-11-11T17:21:07.323999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:07.324054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519098069249952:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:07.324073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:07.325698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:07.325931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519104301672486:2048] [node 1] ICS01 socket: 587 reason# EndOfStream
2025-11-11T17:21:07.325969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519100006705100:2048] [node 1] ICP30 unregister session Session# [4:7571519104301672486:2048] VirtualId# [4:7571519104301672477:0]
2025-11-11T17:21:07.326002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519104301672486:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.326103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:07.326151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:07.333809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519098069249952:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:07.412987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:07.413257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519104537418765:2049] [node 2] ICS01 socket: 606 reason# EndOfStream
2025-11-11T17:21:07.413299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519100242451370:2049] [node 2] ICP30 unregister session Session# [3:7571519104537418765:2049] VirtualId# [3:7571519104537418760:0]
2025-11-11T17:21:07.413333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519104537418765:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.413417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:07.413470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:07.416182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:07.416479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519102490101973:2049] [node 2] ICS01 socket: 611 reason# EndOfStream
2025-11-11T17:21:07.416515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519098195134575:2049] [node 2] ICP30 unregister session Session# [5:7571519102490101973:2049] VirtualId# [5:7571519102490101965:0]
2025-11-11T17:21:07.416561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519102490101973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.416647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:07.416696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:07.417024Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:07.417217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519108596639788:2049] [node 2] ICS01 socket: 608 reason# EndOfStream
2025-11-11T17:21:07.417772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519100006705101:2049] [node 2] ICP30 unregister session Session# [4:7571519108596639788:2049] VirtualId# [4:7571519104301672489:0]
2025-11-11T17:21:07.417824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519108596639788:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.417887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:07.418033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:07.421779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:07.422073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519106659184645:2049] [node 2] ICS01 socket: 628 reason# EndOfStream
2025-11-11T17:21:07.422127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519098069249953:2049] [node 2] ICP30 unregister session Session# [6:7571519106659184645:2049] VirtualId# [6:7571519106659184642:0]
2025-11-11T17:21:07.422170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519106659184645:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.422296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:07.422375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:07.502572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:07.502843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519102490101967:2050] [node 3] ICS01 socket: 565 reason# EndOfStream
2025-11-11T17:21:07.502887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519098195134576:2050] [node 3] ICP30 unregister session Session# [5:7571519102490101967:2050] VirtualId# [5:7571519102490101962:0]
2025-11-11T17:21:07.502915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519102490101967:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.502993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:07.503052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:07.501714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:07.501954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519108596639794:2050] [node 3] ICS01 socket: 568 reason# EndOfStream
2025-11-11T17:21:07.501997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519100006705102:2050] [node 3] ICP30 unregister session Session# [4:7571519108596639794:2050] VirtualId# [4:7571519108596639791:0]
2025-11-11T17:21:07.502034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519108596639794:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.502158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:07.502201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:07.509847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:07.510079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519102364217342:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-11-11T17:21:07.510134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519098069249954:2050] [node 3] ICP30 unregister session Session# [6:7571519102364217342:2050] VirtualId# [6:7571519102364217320:0]
2025-11-11T17:21:07.510168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519102364217342:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.510287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:07.510344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:07.600065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:07.600332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519102490101959:2066] [node 4] ICS01 socket: 585 reason# EndOfStream
2025-11-11T17:21:07.600374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519102490101894:2066] [node 4] ICP30 unregister session Session# [5:7571519102490101959:2066] VirtualId# [5:7571519102490101949:0]
2025-11-11T17:21:07.600407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519102490101959:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.600492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:07.600546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:07.598243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:07.598505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519106659184647:2061] [node 4] ICS01 socket: 619 reason# EndOfStream
2025-11-11T17:21:07.598554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519102364217266:2061] [node 4] ICP30 unregister session Session# [6:7571519106659184647:2061] VirtualId# [6:7571519106659184640:0]
2025-11-11T17:21:07.598587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519106659184647:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.598690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:07.598743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:07.669766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:07.670026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519102364217340:2067] [node 5] ICS01 socket: 614 reason# EndOfStream
2025-11-11T17:21:07.670066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519102364217315:2067] [node 5] ICP30 unregister session Session# [6:7571519102364217340:2067] VirtualId# [6:7571519102364217333:0]
2025-11-11T17:21:07.670100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519102364217340:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:07.670215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:07.670259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:07.766616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519110099885384:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:07.766700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519110099885385:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:07.795992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519107662514827:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:07.796240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519107662514826:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:07.817955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519110188589240:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:07.818006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519110188589241:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:07.858273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519109929753604:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:07.858316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519109929753605:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:07.858346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519109929753606:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:07.889978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519108376692645:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:07.890022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519108376692646:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:07.895825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519108376692644:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:07.914297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519109610622960:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:07.914343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519109610622961:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:07.914373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519109610622962:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:08.426103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519114483556599:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:08.436265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519112671659989:2100] [node 6] ICP01 ready to work
2025-11-11T17:21:08.406474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519114224720961:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:08.498892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519112671659995:2086] [node 4] ICP01 ready to work
2025-11-11T17:21:08.491626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519114394852743:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:08.497928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519114394852744:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:08.498697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519114394852745:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:08.514328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519111957482185:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:08.532770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519114224720964:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:08.514803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519114483556600:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:08.559760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519110099885384:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(115742758ns) {Configure:104(115729059ns) {ProcessConfigured:135(102229ns) {StartInitialHandshake:157(99502ns) {DropHandshakes:469(3219ns) {DropIncomingHandshake:441(783ns) DropOutgoingHandshake:456(759ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:476(2334ns) {SwitchToState:178(838ns)}}}}}
2025-11-11T17:21:08.537909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519111957482186:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.580918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519113905590291:2069] [node 4] ICP01 ready to work
2025-11-11T17:21:08.545872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519114483556601:2107] [node 6] ICP01 ready to work
2025-11-11T17:21:08.611762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519110188589240:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(119788142ns) {Configure:104(119783405ns) {ProcessConfigured:135(77263ns) {StartInitialHandshake:157(74385ns) {DropHandshakes:469(3746ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(915ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:476(2335ns) {SwitchToState:178(782ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.579608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519107662514827:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(227488482ns) {Configure:104(227484812ns) {ProcessConfigured:135(94241ns) {StartInitialHandshake:157(91100ns) {DropHandshakes:469(3715ns) {DropIncomingHandshake:441(804ns) DropOutgoingHandshake:456(672ns)} GenerateSessionVirtualId:988(1151ns) PrepareNewSessionHandshake:476(1938ns) {SwitchToState:178(686ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.631182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519114224720961:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(186459558ns) {Configure:104(186455588ns) {ProcessConfigured:135(61823ns) {StartInitialHandshake:157(59366ns) {DropHandshakes:469(3193ns) {DropIncomingHandshake:441(608ns) DropOutgoingHandshake:456(484ns)} GenerateSessionVirtualId:988(926ns) PrepareNewSessionHandshake:476(2792ns) {SwitchToState:178(1016ns)}}}}}
2025-11-11T17:21:08.631735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519114224720967:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-11T17:21:08.594503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519113905590304:2095] [node 5] ICP01 ready to work
2025-11-11T17:21:08.615911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519114483556602:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12606
2025-11-11T17:21:08.581387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519111957482190:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:08.604828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519111957482194:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3101
2025-11-11T17:21:08.626947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519114394852752:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18993
2025-11-11T17:21:08.629630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519113905590322:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.656489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519112671659989:2100] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(219871073ns) {Configure:104(219863668ns) {ProcessConfigured:135(8026539ns) {StartInitialHandshake:157(8023446ns) {DropHandshakes:469(5843ns) {DropIncomingHandshake:441(875ns) DropOutgoingHandshake:456(589ns)} GenerateSessionVirtualId:988(1496ns) PrepareNewSessionHandshake:476(7208ns) {SwitchToState:178(3494ns)}}}}}
2025-11-11T17:21:08.657160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519112671660007:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:08.657216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519112671660008:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:08.675918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519111957482190:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(62028207ns) {Configure:104(62021445ns) {ProcessConfigured:135(96903ns) {StartInitialHandshake:157(94244ns) {DropHandshakes:469(4350ns) {DropIncomingHandshake:441(957ns) DropOutgoingHandshake:456(653ns)} GenerateSessionVirtualId:988(1438ns) PrepareNewSessionHandshake:476(2340ns) {SwitchToState:178(763ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.681726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519114483556599:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(201410323ns) {Configure:104(201404511ns) {ProcessConfigured:135(14301997ns) {StartInitialHandshake:157(14298140ns) {DropHandshakes:469(2875ns) {DropIncomingHandshake:441(956ns) DropOutgoingHandshake:456(299ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:476(5404ns) {SwitchToState:178(2983ns)}}}}}
2025-11-11T17:21:08.685492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519112671660011:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:08.691313Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519114483556602:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.703716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519114224720969:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-11T17:21:08.691746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519112671659995:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(192540089ns) {Configure:104(192535980ns) {ProcessConfigured:135(9738879ns) {StartInitialHandshake:157(9735400ns) {DropHandshakes:469(4029ns) {DropIncomingHandshake:441(1069ns) DropOutgoingHandshake:456(796ns)} GenerateSessionVirtualId:988(1226ns) PrepareNewSessionHandshake:476(5149ns) {SwitchToState:178(3093ns)}}}}}
2025-11-11T17:21:08.702983Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482194:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.737976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519110188589241:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(246286468ns) {Configure:104(246277818ns) {ProcessConfigured:135(17449496ns) {StartInitialHandshake:157(17446333ns) {DropHandshakes:469(2288ns) {DropIncomingHandshake:441(405ns) DropOutgoingHandshake:456(364ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:476(6817ns) {SwitchToState:178(3651ns)}}}}}
2025-11-11T17:21:08.738222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519110188589241:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.709943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519108376692646:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(358855796ns) {Configure:104(358849412ns) {ProcessConfigured:135(7276009ns) {StartInitialHandshake:157(7272522ns) {DropHandshakes:469(4137ns) {DropIncomingHandshake:441(1012ns) DropOutgoingHandshake:456(739ns)} GenerateSessionVirtualId:988(1452ns) PrepareNewSessionHandshake:476(5036ns) {SwitchToState:178(2090ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.746424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519114483556613:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12606
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.723012Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852752:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.762382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519114394852743:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(194658705ns) {Configure:104(194650690ns) {ProcessConfigured:135(8913651ns) {StartInitialHandshake:157(8911638ns) {DropHandshakes:469(908ns) {DropIncomingHandshake:441(222ns) DropOutgoingHandshake:456(149ns)} GenerateSessionVirtualId:988(654ns) PrepareNewSessionHandshake:476(5134ns) {SwitchToState:178(1111ns)}}}}}
2025-11-11T17:21:08.765991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519110099885385:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(321613097ns) {Configure:104(321606511ns) {ProcessConfigured:135(41326642ns) {StartInitialHandshake:157(41322698ns) {DropHandshakes:469(2272ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(389ns)} GenerateSessionVirtualId:988(374ns) PrepareNewSessionHandshake:476(3900ns) {SwitchToState:178(766ns)}}}}}
2025-11-11T17:21:08.769848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482192:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.769896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519111957482192:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:08.770287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519107662514826:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(250686297ns) {Configure:104(250676453ns) {ProcessConfigured:135(29194259ns) {StartInitialHandshake:157(29190650ns) {DropHandshakes:469(3790ns) {DropIncomingHandshake:441(798ns) DropOutgoingHandshake:456(676ns)} GenerateSessionVirtualId:988(1683ns) PrepareNewSessionHandshake:476(5031ns) {SwitchToState:178(2750ns)}}}}}
2025-11-11T17:21:08.721927Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519114224720967:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.722332Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519112671660008:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.725734Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519113905590322:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.786774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519111957482200:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3101
2025-11-11T17:21:08.787474Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519111957482192:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:08.787836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519107662514827:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.787890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:08.787935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519107662514827:2049] [node 3] ICP22 created new session: [2:7571519111957482202:2049]
2025-11-11T17:21:08.787986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519111957482202:2049] [node 3] ICS09 handshake done sender: [2:7571519111957482192:2111] self: [2:7571519111957482191:0] peer: [3:7571519114483556611:0] socket: 572 qp: -1
2025-11-11T17:21:08.788017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519111957482202:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:08.788058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:08.769122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519114483556614:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12606
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.790166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482188:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.790212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519111957482188:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:08.797820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519107662514826:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.769192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519114483556615:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12606
2025-11-11T17:21:08.767185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519109929753606:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(373199309ns) {Configure:104(373193480ns) {ProcessConfigured:135(41281891ns) {StartInitialHandshake:157(24748984ns) {DropHandshakes:469(2353ns) {DropIncomingHandshake:441(360ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(1049ns) PrepareNewSessionHandshake:476(2936ns) {SwitchToState:178(972ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.746823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519109610622961:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(373944561ns) {Configure:104(373938064ns) {ProcessConfigured:135(6948631ns) {StartInitialHandshake:157(6945729ns) {DropHandshakes:469(4027ns) {DropIncomingHandshake:441(974ns) DropOutgoingHandshake:456(731ns)} GenerateSessionVirtualId:988(1899ns) PrepareNewSessionHandshake:476(3483ns) {SwitchToState:178(460ns)}}}}}
2025-11-11T17:21:08.766645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519114394852758:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18993
2025-11-11T17:21:08.722463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519112671659995:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.739063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519108376692644:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(245159016ns) {Configure:104(245152326ns) {ProcessConfigured:135(18547563ns) {StartInitialHandshake:157(18545238ns) {DropHandshakes:469(678ns) {DropIncomingHandshake:441(151ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:476(3046ns) {SwitchToState:178(335ns)}}}}}
2025-11-11T17:21:08.767920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519114224720976:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.796474Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519114483556613:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.768695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519114394852761:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18993
2025-11-11T17:21:08.796774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519114483556616:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12606
2025-11-11T17:21:08.797990Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519114483556614:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.805723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519110188589240:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.817990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519114483556600:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(251419503ns) {Configure:104(251411263ns) {ProcessConfigured:135(20028324ns) {StartInitialHandshake:157(20024790ns) {DropHandshakes:469(2949ns) {DropIncomingHandshake:441(675ns) DropOutgoingHandshake:456(719ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:476(7248ns) {SwitchToState:178(3960ns)}}}}}
2025-11-11T17:21:08.747305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519109610622961:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.806460Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519111957482188:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:08.777800Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519114224720966:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.879371Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519114483556602:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:08.777840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519114224720966:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.754074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519113905590330:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.806653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519111957482185:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(238998062ns) {Configure:104(238992328ns) {ProcessConfigured:135(15431428ns) {StartInitialHandshake:157(7967929ns) {DropHandshakes:469(2102ns) {DropIncomingHandshake:441(141ns) DropOutgoingHandshake:456(278ns)} GenerateSessionVirtualId:988(1644ns) PrepareNewSessionHandshake:476(3194ns) {SwitchToState:178(1607ns)}}}}}
2025-11-11T17:21:08.741751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519112671660011:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.779347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519109929753604:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(296368954ns) {Configure:104(296359001ns) {ProcessConfigured:135(21665654ns) {StartInitialHandshake:157(21663749ns) {DropHandshakes:469(1059ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(170ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:476(3811ns) {SwitchToState:178(266ns)}}}}}
2025-11-11T17:21:08.781509Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852747:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.890690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519114483556615:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.781534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519114394852747:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.909782Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:08.822026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519107662514826:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(27501862ns) {IncomingHandshake:261(27499109ns) {DropIncomingHandshake:441(2031ns) PrepareNewSessionHandshake:476(5747ns) {SwitchToState:178(3731ns)} GenerateSessionVirtualId:988(428ns) IssueIncomingHandshakeReply:193(6664687ns) {SwitchToState:178(1858ns)}}}
2025-11-11T17:21:08.780860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519109929753604:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.779094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519112671660007:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.793658Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519113905590330:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.842562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519111957482209:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3101
2025-11-11T17:21:08.918838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519110188589241:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:08.812676Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:08.794986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519113905590304:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.802371Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852750:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.810048Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519114224720966:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.868577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519111957482186:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.918902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:08.810822Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519114224720969:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.802401Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519114394852750:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.813816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519112671659995:2086] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519112671660008:2112] held: no
2025-11-11T17:21:08.806119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519109610622960:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(317912553ns) {Configure:104(317898450ns) {ProcessConfigured:135(42994508ns) {StartInitialHandshake:157(42990819ns) {DropHandshakes:469(3347ns) {DropIncomingHandshake:441(853ns) DropOutgoingHandshake:456(635ns)} GenerateSessionVirtualId:988(1067ns) PrepareNewSessionHandshake:476(4829ns) {SwitchToState:178(2257ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.803295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519114394852763:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18993
2025-11-11T17:21:08.874461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519113905590291:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(266499586ns) {Configure:104(266488790ns) {ProcessConfigured:135(27973113ns) {StartInitialHandshake:157(27969397ns) {DropHandshakes:469(4110ns) {DropIncomingHandshake:441(924ns) DropOutgoingHandshake:456(1180ns)} GenerateSessionVirtualId:988(1116ns) PrepareNewSessionHandshake:476(3513ns) {SwitchToState:178(764ns)}}}}}
2025-11-11T17:21:08.870121Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482209:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.918953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519110188589241:2049] [node 2] ICP22 created new session: [3:7571519114483556624:2049]
2025-11-11T17:21:08.810868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519114224720961:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.814089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519112671660023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:08.818700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519114394852745:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(227379543ns) {Configure:104(227371174ns) {ProcessConfigured:135(29614808ns) {StartInitialHandshake:157(29612918ns) {DropHandshakes:469(699ns) {DropIncomingHandshake:441(137ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:476(1901ns) {SwitchToState:178(630ns)}}}}}
2025-11-11T17:21:08.877931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:08.810944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:08.919024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519114483556624:2049] [node 2] ICS09 handshake done sender: [3:7571519114483556602:2110] self: [3:7571519114483556611:0] peer: [2:7571519111957482191:0] socket: 573 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.906041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519113905590337:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.810996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519114224720961:2109] [node 5] ICP22 created new session: [4:7571519114224720982:2109]
2025-11-11T17:21:08.821860Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519114394852750:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.825679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519112671660006:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.919075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519114483556624:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:08.878010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519111957482186:2108] [node 6] ICP22 created new session: [2:7571519111957482211:2108]
2025-11-11T17:21:08.921694Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:08.919120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:08.811045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519114224720982:2109] [node 5] ICS09 handshake done sender: [4:7571519114224720966:2110] self: [4:7571519114224720965:0] peer: [5:7571519112671660018:0] socket: 581 qp: -1
2025-11-11T17:21:08.825728Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519112671660006:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:08.825683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852758:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.842093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852757:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.842137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519114394852757:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:08.843093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852761:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.843146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519110099885384:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.878058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519111957482211:2108] [node 6] ICS09 handshake done sender: [2:7571519111957482188:2110] self: [2:7571519111957482187:0] peer: [6:7571519113905590328:0] socket: 571 qp: -1
2025-11-11T17:21:08.811081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519114224720982:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:08.838011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519108376692645:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(313745828ns) {Configure:104(313741929ns) {ProcessConfigured:135(33313129ns) {StartInitialHandshake:157(33309559ns) {DropHandshakes:469(629ns) {DropIncomingHandshake:441(203ns) DropOutgoingHandshake:456(86ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(3931ns) {SwitchToState:178(742ns)}}}}}
2025-11-11T17:21:08.922428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519109610622961:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519113905590322:2110] held: no
2025-11-11T17:21:08.843223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:08.843274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519110099885384:2048] [node 2] ICP22 created new session: [1:7571519114394852766:2048]
2025-11-11T17:21:08.843324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519114394852766:2048] [node 2] ICS09 handshake done sender: [1:7571519114394852750:2111] self: [1:7571519114394852749:0] peer: [2:7571519111957482205:0] socket: 574 qp: -1
2025-11-11T17:21:08.843362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519114394852766:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:08.843399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:08.844758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519114394852771:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18993
2025-11-11T17:21:08.845244Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519114394852747:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:08.845520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519114394852744:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.845573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:08.845635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519114394852744:2107] [node 4] ICP22 created new session: [1:7571519114394852772:2107]
2025-11-11T17:21:08.845683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519114394852772:2107] [node 4] ICS09 handshake done sender: [1:7571519114394852747:2110] self: [1:7571519114394852746:0] peer: [4:7571519114224720979:0] socket: 570 qp: -1
2025-11-11T17:21:08.845713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519114394852772:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:08.845751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:08.861699Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852763:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.863013Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519114394852757:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:08.863313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519110099885385:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.863387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:08.863453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519110099885385:2049] [node 3] ICP22 created new session: [1:7571519114394852775:2049]
2025-11-11T17:21:08.863502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519114394852775:2049] [node 3] ICS09 handshake done sender: [1:7571519114394852757:2114] self: [1:7571519114394852753:0] peer: [3:7571519114483556618:0] socket: 598 qp: -1
2025-11-11T17:21:08.863540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519114394852775:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:08.863581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:08.868326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519110099885384:2048] [node 2] ICP18 handshake (actor [1:7571519114394852763:2118]) is too old
2025-11-11T17:21:08.885813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852771:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.917887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519114394852745:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(19878744ns) {IncomingHandshake:261(19876365ns) {DropIncomingHandshake:441(949ns) PrepareNewSessionHandshake:476(1835ns) {SwitchToState:178(988ns)} GenerateSessionVirtualId:988(949ns) IssueIncomingHandshakeReply:193(9607ns)}}
2025-11-11T17:21:09.023414Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852756:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.023469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519114394852756:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:09.023701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519114394852762:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.023732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519114394852762:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:09.035894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519114394852762:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:09.036854Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519114394852756:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:09.041679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519114394852745:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:09.048828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519114394852743:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:09.057893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:09.065398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519114394852745:2108] [node 5] ICP22 created new session: [1:7571519118689820077:2108]
2025-11-11T17:21:09.066001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:09.070325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519118689820077:2108] [node 5] ICS09 handshake done sender: [1:7571519114394852762:2117] self: [1:7571519114394852754:0] peer: [5:7571519112671660038:0] socket: 623 qp: -1
2025-11-11T17:21:09.070334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519114394852743:2109] [node 6] ICP22 created new session: [1:7571519118689820078:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.919857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519110188589240:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519114483556614:2115] held: no
2025-11-11T17:21:08.878099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519111957482211:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:08.811122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:08.874890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519108376692645:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:09.078823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519118689820078:2109] [node 6] ICS09 handshake done sender: [1:7571519114394852756:2113] self: [1:7571519114394852755:0] peer: [6:7571519113905590343:0] socket: 618 qp: -1
2025-11-11T17:21:08.922964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519113905590339:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
2025-11-11T17:21:08.920044Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519114483556607:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.878151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.923098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519113905590340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
2025-11-11T17:21:08.889847Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519112671660006:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:09.082104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519118689820077:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:08.811837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519114224720961:2109] [node 5] ICP18 handshake (actor [4:7571519114224720969:2112]) is too old
2025-11-11T17:21:08.878526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519111957482186:2108] [node 6] ICP18 handshake (actor [2:7571519111957482209:2117]) is too old
2025-11-11T17:21:08.913942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519112671659995:2086] [node 4] ICP35 event processing took too much time PendingConnection:278(24014912ns) {HandleHandshakeStatus:395(24010950ns) {LogHandshakeFail:472(14253ns) {UpdateErrorStateLog:544(5454ns)} DropOutgoingHandshake:456(655ns) ProcessPendingSessionEvents:495(23935523ns) {DropSessionEvent:512(10001ns) DropSessionEvent:512(8027942ns) DropSessionEvent:512(10207ns) DropSessionEvent:512(4509ns) DropSessionEvent:512(5172ns) DropSessionEvent:512(3940541ns) DropSessionEvent:512(21018ns) DropSessionEvent:512(5836ns) DropSessionEvent:512(5471ns) DropSessionEvent:512(11892125ns)}}}
2025-11-11T17:21:08.925914Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519112671660008:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:08.937733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519112671659995:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:08.937795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:08.937849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519112671659995:2086] [node 4] ICP22 created new session: [5:7571519112671660027:2086]
2025-11-11T17:21:08.937934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519112671660027:2086] [node 4] ICS09 handshake done sender: [5:7571519112671660008:2112] self: [5:7571519112671660018:0] peer: [4:7571519114224720965:0] socket: 583 qp: -1
2025-11-11T17:21:08.945128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519112671660027:2086] [node 4] ICS10 traffic start
2025-11-11T17:21:08.950836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519112671660030:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:08.950894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519112671660031:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:08.920070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519114483556607:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:08.945712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519109610622962:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(321007619ns) {Configure:104(320998814ns) {ProcessConfigured:135(53363237ns) {StartInitialHandshake:157(53359869ns) {DropHandshakes:469(569ns) {DropIncomingHandshake:441(124ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:476(2612ns) {SwitchToState:178(125ns)}}}}}
2025-11-11T17:21:08.950990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519112671660032:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:08.953670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:08.954539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519112671659989:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.954663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:08.954714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519112671659989:2100] [node 6] ICP22 created new session: [5:7571519112671660033:2100]
2025-11-11T17:21:08.954760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519112671660033:2100] [node 6] ICS09 handshake done sender: [5:7571519112671660006:2110] self: [5:7571519112671660005:0] peer: [6:7571519113905590333:0] socket: 584 qp: -1
2025-11-11T17:21:08.954796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519112671660033:2100] [node 6] ICS10 traffic start
2025-11-11T17:21:08.954832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:08.965671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519112671660032:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.967520Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519112671660011:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:08.972528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519108376692646:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.973037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519112671659995:2086] [node 4] ICP35 event processing took too much time PendingConnection:278(30005180ns) {HandleHandshakeStatus:329(30001743ns) {DropIncomingHandshake:441(1151ns) DropOutgoingHandshake:456(540ns) SwitchToState:178(772ns) ProcessPendingSessionEvents:495(856ns)}}
2025-11-11T17:21:08.981000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519112671660030:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.985811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519108376692645:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:08.985914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:08.985970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519108376692645:2049] [node 2] ICP22 created new session: [5:7571519112671660036:2049]
2025-11-11T17:21:08.986023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519112671660036:2049] [node 2] ICS09 handshake done sender: [5:7571519112671660011:2113] self: [5:7571519112671660025:0] peer: [2:7571519111957482195:0] socket: 586 qp: -1
2025-11-11T17:21:08.986063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519112671660036:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:08.986102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:08.992028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519108376692644:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:09.035659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519116966627335:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:09.057687Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519112671660030:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:09.063322Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:09.064211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519108376692644:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519112671660030:2119] held: no
2025-11-11T17:21:09.069767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519108376692644:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.073756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:09.073830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519108376692644:2048] [node 1] ICP22 created new session: [5:7571519116966627337:2048]
2025-11-11T17:21:09.073879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519116966627337:2048] [node 1] ICS09 handshake done sender: [5:7571519112671660030:2119] self: [5:7571519112671660038:0] peer: [1:7571519114394852754:0] socket: 626 qp: -1
2025-11-11T17:21:09.073923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519116966627337:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:09.073965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:09.078191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519116966627336:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-11-11T17:21:09.089754Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15921) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:09.089907Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519112671660032:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:09.094604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519108376692646:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15921) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519112671660032:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.920781Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:08.930401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519110188589241:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:08.938271Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519114483556607:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:08.942185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519114483556622:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12606
2025-11-11T17:21:08.952100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519114483556601:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(304075398ns) {Configure:104(296528896ns) {ProcessConfigured:135(39438686ns) {StartInitialHandshake:157(39435802ns) {DropHandshakes:469(3845ns) {DropIncomingHandshake:441(891ns) DropOutgoingHandshake:456(797ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:476(4230ns) {SwitchToState:178(1858ns)}}}}}
2025-11-11T17:21:08.958322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519114483556599:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.960423Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519114483556614:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:08.965578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:08.965675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519114483556599:2108] [node 4] ICP22 created new session: [3:7571519114483556629:2108]
2025-11-11T17:21:08.970098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519110188589240:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:08.970144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:08.970200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519110188589240:2048] [node 1] ICP22 created new session: [3:7571519114483556631:2048]
2025-11-11T17:21:08.970244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519114483556631:2048] [node 1] ICS09 handshake done sender: [3:7571519114483556614:2115] self: [3:7571519114483556618:0] peer: [1:7571519114394852753:0] socket: 603 qp: -1
2025-11-11T17:21:08.970282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519114483556631:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:08.970319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:08.979599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519114483556629:2108] [node 4] ICS09 handshake done sender: [3:7571519114483556607:2112] self: [3:7571519114483556605:0] peer: [4:7571519114224720987:0] socket: 592 qp: -1
2025-11-11T17:21:08.990164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519114483556629:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:08.998951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:09.022290Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519114483556619:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.022336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519114483556619:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:09.026731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519118778523930:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12606
2025-11-11T17:21:09.046488Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519114483556599:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(88181824ns) {HandleHandshakeStatus:329(88178919ns) {DropIncomingHandshake:441(15487ns) DropOutgoingHandshake:456(600ns) SwitchToState:178(846ns) ProcessPendingSessionEvents:495(19128766ns) {ForwardSessionEventToSession:612(19072469ns) ForwardSessionEventToSession:612(7299ns) ForwardSessionEventToSession:612(3116ns) ForwardSessionEventToSession:612(6756ns) ForwardSessionEventToSession:612(2929ns) ForwardSessionEventToSession:612(5572ns) ForwardSessionEventToSession:612(8045ns) ForwardSessionEventToSession:612(2918ns) ForwardSessionEventToSession:612(6058ns) ForwardSessionEventToSession:612(2616ns)}}}
2025-11-11T17:21:09.048184Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519114483556619:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:09.065349Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519114483556623:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.065392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519114483556623:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:09.058171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519114483556600:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:09.078448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:09.082179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519114483556600:2109] [node 5] ICP22 created new session: [3:7571519118778523933:2109]
2025-11-11T17:21:09.085071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519114483556623:2120] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:09.085447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519118778523933:2109] [node 5] ICS09 handshake done sender: [3:7571519114483556619:2118] self: [3:7571519114483556617:0] peer: [5:7571519112671660035:0] socket: 591 qp: -1
2025-11-11T17:21:09.086536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519118778523933:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:09.094798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:09.105762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519114483556601:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:09.082307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519118689820078:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:08.887072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519111957482213:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3101
2025-11-11T17:21:08.833978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519114224720964:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(266624735ns) {Configure:104(266619651ns) {ProcessConfigured:135(24046176ns) {StartInitialHandshake:157(24038373ns) {DropHandshakes:469(4303ns) {DropIncomingHandshake:441(1016ns) DropOutgoingHandshake:456(1048ns)} GenerateSessionVirtualId:988(1215ns) PrepareNewSessionHandshake:476(4928ns) {SwitchToState:178(2152ns)}}}}}
2025-11-11T17:21:08.841997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519109929753605:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(304355512ns) {Configure:104(304350903ns) {ProcessConfigured:135(54001038ns) {StartInitialHandshake:157(53998650ns) {DropHandshakes:469(3693ns) {DropIncomingHandshake:441(841ns) DropOutgoingHandshake:456(672ns)} GenerateSessionVirtualId:988(940ns) PrepareNewSessionHandshake:476(3310ns) {SwitchToState:178(279ns)}}}}}
2025-11-11T17:21:08.874569Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:08.885982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519114224720976:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.893727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519109929753604:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519114224720967:2111] held: no
2025-11-11T17:21:08.894457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519109929753606:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.894779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519114224720986:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-11T17:21:08.957177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519109929753604:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32130432ns) {HandleHandshakeStatus:395(32124590ns) {LogHandshakeFail:472(24103ns) {UpdateErrorStateLog:544(9107ns)} DropOutgoingHandshake:456(2438ns) ProcessPendingSessionEvents:495(16001649ns) {DropSessionEvent:512(12770ns) DropSessionEvent:512(7672ns) DropSessionEvent:512(5490ns) DropSessionEvent:512(15930934ns) DropSessionEvent:512(6342ns) DropSessionEvent:512(6022ns) DropSessionEvent:512(7237ns) DropSessionEvent:512(4632ns) DropSessionEvent:512(5776ns) DropSessionEvent:512(5365ns)}}}
2025-11-11T17:21:08.960843Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519114224720967:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:08.971130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519114224720989:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-11T17:21:08.979787Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15921) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:08.990614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519109929753606:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15921) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519114224720976:2116] held: no
2025-11-11T17:21:08.990901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519109929753604:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:08.990950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:08.990999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519109929753604:2048] [node 1] ICP22 created new session: [4:7571519114224720991:2048]
2025-11-11T17:21:08.991050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519114224720991:2048] [node 1] ICS09 handshake done sender: [4:7571519114224720967:2111] self: [4:7571519114224720979:0] peer: [1:7571519114394852746:0] socket: 580 qp: -1
2025-11-11T17:21:08.991084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519114224720991:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:08.991117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:09.000819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519114224720989:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.009815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519109929753605:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:09.046065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519118519688290:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-11T17:21:09.050673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519114224720990:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-11T17:21:09.065743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519114224720985:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.065777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519114224720985:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:09.078397Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519114224720976:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:09.086550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519109929753606:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.093895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519118519688291:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-11T17:21:09.097931Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:09.106376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:09.106716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519109929753605:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519114224720989:2119] held: no
2025-11-11T17:21:09.113640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519109929753606:2050] [node 3] ICP22 created new session: [4:7571519118519688292:2050]
2025-11-11T17:21:09.115370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519118519688292:2050] [node 3] ICS09 handshake done sender: [4:7571519114224720976:2116] self: [4:7571519114224720987:0] peer: [3:7571519114483556605:0] socket: 599 qp: -1
2025-11-11T17:21:09.124026Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519114224720985:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:08.945913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519113905590341:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
2025-11-11T17:21:08.906694Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:09.082382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:09.138186Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519108376692646:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(23728786ns) {HandleHandshakeStatus:395(23725263ns) {LogHandshakeFail:472(2527315ns) {UpdateErrorStateLog:544(5725ns)} DropOutgoingHandshake:456(3067ns) ProcessPendingSessionEvents:495(15650529ns) {DropSessionEvent:512(36460ns) DropSessionEvent:512(5896ns) DropSessionEvent:512(5279ns) DropSessionEvent:512(4127381ns) DropSessionEvent:512(13239ns) DropSessionEvent:512(5058ns) DropSessionEvent:512(6547ns) DropSessionEvent:512(11428771ns) DropSessionEvent:512(7044ns) DropSessionEvent:512(5569ns)}}}
2025-11-11T17:21:09.142641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519108376692646:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.123309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519114483556600:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(64934806ns) {HandleHandshakeStatus:329(64931657ns) {DropIncomingHandshake:441(36450ns) DropOutgoingHandshake:456(5574ns) SwitchToState:178(667ns) ProcessPendingSessionEvents:495(7771401ns) {ForwardSessionEventToSession:612(7723539ns) ForwardSessionEventToSession:612(5943ns) ForwardSessionEventToSession:612(5834ns) ForwardSessionEventToSession:612(4532ns) ForwardSessionEventToSession:612(2889ns) ForwardSessionEventToSession:612(4177ns) ForwardSessionEventToSession:612(2719ns) ForwardSessionEventToSession:612(4562ns) ForwardSessionEventToSession:612(3617ns) ForwardSessionEventToSession:612(3483ns)}}}
2025-11-11T17:21:08.966082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519113905590339:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.133845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519109929753605:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17016671ns) {HandleHandshakeStatus:395(17014214ns) {LogHandshakeFail:472(7580ns) {UpdateErrorStateLog:544(2513ns)} DropOutgoingHandshake:456(1516ns) ProcessPendingSessionEvents:495(8475006ns) {DropSessionEvent:512(10732ns) DropSessionEvent:512(5767ns) DropSessionEvent:512(4589ns) DropSessionEvent:512(5946ns) DropSessionEvent:512(4504ns) DropSessionEvent:512(5050ns) DropSessionEvent:512(5088ns) DropSessionEvent:512(1522703ns) DropSessionEvent:512(4905ns) DropSessionEvent:512(6803ns)}}}
2025-11-11T17:21:09.133963Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519114224720989:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:09.137634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519118519688292:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:09.137712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:09.149738Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519118519688290:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.910252Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482213:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.084448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519114394852743:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(35418237ns) {HandleHandshakeStatus:329(35415536ns) {DropIncomingHandshake:441(13816ns) DropOutgoingHandshake:456(807ns) SwitchToState:178(367ns) ProcessPendingSessionEvents:495(1729048ns) {ForwardSessionEventToSession:612(51251ns) ForwardSessionEventToSession:612(7199ns) ForwardSessionEventToSession:612(3034ns) ForwardSessionEventToSession:612(2892ns) ForwardSessionEventToSession:612(5676ns) ForwardSessionEventToSession:612(3439ns) ForwardSessionEventToSession:612(11337ns) ForwardSessionEventToSession:612(3158ns) ForwardSessionEventToSession:612(5583ns) ForwardSessionEventToSession:612(1625883ns)}}}
2025-11-11T17:21:09.134219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:09.149896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519109929753606:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(64356240ns) {HandleHandshakeStatus:329(64353577ns) {DropIncomingHandshake:441(1135ns) DropOutgoingHandshake:456(616ns) SwitchToState:178(711ns) ProcessPendingSessionEvents:495(1082ns)}}
2025-11-11T17:21:08.966300Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16919) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:09.156847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:09.086505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:08.910390Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482200:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.138320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519114483556601:2107] [node 6] ICP22 created new session: [3:7571519118778523935:2107]
2025-11-11T17:21:09.150604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519109929753605:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:08.969814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519113905590304:2095] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16919) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519113905590330:2113] held: no
2025-11-11T17:21:09.110440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519114394852745:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(68476863ns) {HandleHandshakeStatus:329(68473620ns) {DropIncomingHandshake:441(27810ns) DropOutgoingHandshake:456(636ns) SwitchToState:178(585ns) ProcessPendingSessionEvents:495(8073848ns) {ForwardSessionEventToSession:612(8014751ns) ForwardSessionEventToSession:612(7487ns) ForwardSessionEventToSession:612(5952ns) ForwardSessionEventToSession:612(3477ns) ForwardSessionEventToSession:612(5629ns) ForwardSessionEventToSession:612(5752ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(5389ns) ForwardSessionEventToSession:612(3468ns) ForwardSessionEventToSession:612(3156ns)}}}
2025-11-11T17:21:09.157751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519108376692646:2050] [node 3] ICP22 created new session: [5:7571519116966627339:2050]
2025-11-11T17:21:09.146893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519118778523935:2107] [node 6] ICS09 handshake done sender: [3:7571519114483556623:2120] self: [3:7571519114483556621:0] peer: [6:7571519118200557645:0] socket: 578 qp: -1
2025-11-11T17:21:09.151450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519114224720964:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.910460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519107662514826:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519111957482194:2112] held: no
2025-11-11T17:21:09.146948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519118778523935:2107] [node 6] ICS10 traffic start
2025-11-11T17:21:09.157551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:09.182367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519114483556601:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(76314795ns) {HandleHandshakeStatus:329(76311173ns) {DropIncomingHandshake:441(956ns) DropOutgoingHandshake:456(733ns) SwitchToState:178(691ns) ProcessPendingSessionEvents:495(8478879ns) {ForwardSessionEventToSession:612(8417173ns) ForwardSessionEventToSession:612(8255ns) ForwardSessionEventToSession:612(2802ns) ForwardSessionEventToSession:612(7099ns) ForwardSessionEventToSession:612(3401ns) ForwardSessionEventToSession:612(5517ns) ForwardSessionEventToSession:612(5993ns) ForwardSessionEventToSession:612(2898ns) ForwardSessionEventToSession:612(8394ns) ForwardSessionEventToSession:612(2575ns)}}}
2025-11-11T17:21:08.978574Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519113905590330:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:08.994053Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519113905590322:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:08.997881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519109610622960:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:08.998336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519113905590342:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
2025-11-11T17:21:09.005304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519109610622961:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.021697Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519113905590341:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.023326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519113905590304:2095] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.023965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:09.037871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519109610622961:2049] [node 2] ICP22 created new session: [6:7571519118200557642:2049]
2025-11-11T17:21:09.041175Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519113905590342:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.041401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519118200557644:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
2025-11-11T17:21:09.042169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519113905590291:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:09.044056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:09.048796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519113905590304:2095] [node 5] ICP22 created new session: [6:7571519118200557646:2095]
2025-11-11T17:21:09.049918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519109610622962:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:09.054135Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519113905590339:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:09.054792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519118200557642:2049] [node 2] ICS09 handshake done sender: [6:7571519113905590322:2110] self: [6:7571519113905590328:0] peer: [2:7571519111957482187:0] socket: 577 qp: -1
2025-11-11T17:21:09.057959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519118200557646:2095] [node 5] ICS09 handshake done sender: [6:7571519113905590330:2113] self: [6:7571519113905590333:0] peer: [5:7571519112671660005:0] socket: 593 qp: -1
2025-11-11T17:21:09.058011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519118200557642:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:09.058552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519109610622960:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.063284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:09.065573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519109610622960:2048] [node 1] ICP22 created new session: [6:7571519118200557647:2048]
2025-11-11T17:21:09.070195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519118200557646:2095] [node 5] ICS10 traffic start
2025-11-11T17:21:09.070474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:09.075530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:09.082174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519118200557647:2048] [node 1] ICS09 handshake done sender: [6:7571519113905590339:2118] self: [6:7571519113905590343:0] peer: [1:7571519114394852755:0] socket: 605 qp: -1
2025-11-11T17:21:09.084045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519118200557647:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:09.089841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:09.095062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519113905590304:2095] [node 5] ICP35 event processing took too much time PendingConnection:278(70113229ns) {HandleHandshakeStatus:329(70109919ns) {DropIncomingHandshake:441(1349ns) DropOutgoingHandshake:456(1152ns) SwitchToState:178(1030ns) ProcessPendingSessionEvents:495(1263ns)}}
2025-11-11T17:21:09.113649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519109610622961:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(92455090ns) {HandleHandshakeStatus:329(92450784ns) {DropIncomingHandshake:441(1623ns) DropOutgoingHandshake:456(775ns) SwitchToState:178(742ns) ProcessPendingSessionEvents:495(1511ns)}}
2025-11-11T17:21:09.134098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519118200557651:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
2025-11-11T17:21:09.137871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519118200557652:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16480
2025-11-11T17:21:09.151246Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519113905590341:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:09.156808Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519113905590342:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:09.160972Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:09.161821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519109610622962:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.161963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:09.162016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519109610622962:2050] [node 3] ICP22 created new session: [6:7571519118200557653:2050]
2025-11-11T17:21:09.162064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519118200557653:2050] [node 3] ICS09 handshake done sender: [6:7571519113905590342:2121] self: [6:7571519118200557645:0] peer: [3:7571519114483556621:0] socket: 582 qp: -1
2025-11-11T17:21:09.162098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519118200557653:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:09.162136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:09.162459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519113905590291:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:09.162535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:09.162581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519113905590291:2069] [node 4] ICP22 created new session: [6:7571519118200557655:2069]
2025-11-11T17:21:09.162626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519118200557655:2069] [node 4] ICS09 handshake done sender: [6:7571519113905590341:2120] self: [6:7571519118200557643:0] peer: [4:7571519114224720981:0] socket: 622 qp: -1
2025-11-11T17:21:09.162663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519118200557655:2069] [node 4] ICS10 traffic start
2025-11-11T17:21:09.162714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:09.162906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519113905590291:2069] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:09.178966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519109610622960:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(87374942ns) {HandleHandshakeStatus:329(87372674ns) {DropIncomingHandshake:441(1077ns) DropOutgoingHandshake:456(6232ns) SwitchToState:178(834ns) ProcessPendingSessionEvents:495(39491745ns) {ForwardSessionEventToSession:612(9019968ns) ForwardSessionEventToSession:612(4448996ns) ForwardSessionEventToSession:612(6377ns) ForwardSessionEventToSession:612(3261ns) ForwardSessionEventToSession:612(11560767ns) ForwardSessionEventToSession:612(6763102ns) ForwardSessionEventToSession:612(16509ns) ForwardSessionEventToSession:612(4486769ns) ForwardSessionEventToSession:612(3387ns) ForwardSessionEventToSession:612(3141383ns)}}}
2025-11-11T17:21:09.157808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519116966627339:2050] [node 3] ICS09 handshake done sender: [5:7571519112671660032:2121] self: [5:7571519112671660035:0] peer: [3:7571519114483556617:0] socket: 630 qp: -1
2025-11-11T17:21:09.158114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519116966627339:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:09.158162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:09.160319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519108376692646:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(21961960ns) {HandleHandshakeStatus:329(17601148ns) {DropIncomingHandshake:441(1040ns) DropOutgoingHandshake:456(517ns) SwitchToState:178(1039ns) ProcessPendingSessionEvents:495(514ns)}}
2025-11-11T17:21:08.910715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519107662514827:2049] [node 3] ICP18 handshake (actor [2:7571519111957482200:2115]) is too old
2025-11-11T17:21:08.913794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519111957482210:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3101
2025-11-11T17:21:08.921661Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482196:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:08.921697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519111957482196:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:08.939004Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519111957482194:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:08.940868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519107662514826:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:08.940957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:08.941008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519107662514826:2048] [node 1] ICP22 created new session: [2:7571519111957482217:2048]
2025-11-11T17:21:08.941053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519111957482217:2048] [node 1] ICS09 handshake done sender: [2:7571519111957482194:2112] self: [2:7571519111957482205:0] peer: [1:7571519114394852749:0] socket: 575 qp: -1
2025-11-11T17:21:08.941091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519111957482217:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:08.941131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:08.951411Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519111957482196:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:08.967583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519111957482190:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:08.978155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519111957482219:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3101
2025-11-11T17:21:08.978466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:08.978520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519111957482190:2109] [node 5] ICP22 created new session: [2:7571519111957482220:2109]
2025-11-11T17:21:09.001746Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:09.023955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519111957482220:2109] [node 5] ICS09 handshake done sender: [2:7571519111957482196:2113] self: [2:7571519111957482195:0] peer: [5:7571519112671660025:0] socket: 585 qp: -1
2025-11-11T17:21:09.026977Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519111957482208:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.027011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519111957482208:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:09.037164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519111957482220:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:09.037229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:09.054583Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519111957482208:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:09.078552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519111957482185:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:09.089907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:09.089961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519111957482185:2107] [node 4] ICP22 created new session: [2:7571519116252449519:2107]
2025-11-11T17:21:09.098071Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519111957482190:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(116144475ns) {HandleHandshakeStatus:329(114416786ns) {DropIncomingHandshake:441(665ns) DropOutgoingHandshake:456(893ns) SwitchToState:178(698ns) ProcessPendingSessionEvents:495(27234664ns) {ForwardSessionEventToSession:612(4048330ns) ForwardSessionEventToSession:612(4399433ns) ForwardSessionEventToSession:612(4043ns) ForwardSessionEventToSession:612(1516310ns) ForwardSessionEventToSession:612(5393301ns) ForwardSessionEventToSession:612(11409ns) ForwardSessionEventToSession:612(8196604ns) ForwardSessionEventToSession:612(4545ns) ForwardSessionEventToSession:612(3108ns) ForwardSessionEventToSession:612(3621684ns)}}}
2025-11-11T17:21:09.098137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519111957482190:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:09.105772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519116252449519:2107] [node 4] ICS09 handshake done sender: [2:7571519111957482208:2116] self: [2:7571519111957482206:0] peer: [4:7571519114224720993:0] socket: 594 qp: -1
2025-11-11T17:21:09.117711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519116252449519:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:09.117777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:09.192573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519111957482185:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(95674517ns) {HandleHandshakeStatus:329(90396320ns) {DropIncomingHandshake:441(27369ns) DropOutgoingHandshake:456(672ns) SwitchToState:178(1208ns) ProcessPendingSessionEvents:495(27422460ns) {ForwardSessionEventToSession:612(18192743ns) ForwardSessionEventToSession:612(15902ns) ForwardSessionEventToSession:612(950866ns) ForwardSessionEventToSession:612(4967ns) ForwardSessionEventToSession:612(511264ns) ForwardSessionEventToSession:612(1722087ns) ForwardSessionEventToSession:612(4188ns) ForwardSessionEventToSession:612(2804ns) ForwardSessionEventToSession:612(5991872ns) ForwardSessionEventToSession:612(4275ns)}}}
2025-11-11T17:21:09.151562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:09.151616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519114224720964:2108] [node 6] ICP22 created new session: [4:7571519118519688296:2108]
2025-11-11T17:21:09.151658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519118519688296:2108] [node 6] ICS09 handshake done sender: [4:7571519114224720985:2117] self: [4:7571519114224720981:0] peer: [6:7571519118200557643:0] socket: 600 qp: -1
2025-11-11T17:21:09.151696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519118519688296:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:09.151729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:09.163535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:09.163587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519109929753605:2049] [node 2] ICP22 created new session: [4:7571519118519688298:2049]
2025-11-11T17:21:09.173742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519118519688298:2049] [node 2] ICS09 handshake done sender: [4:7571519114224720989:2119] self: [4:7571519114224720993:0] peer: [2:7571519111957482206:0] socket: 610 qp: -1
2025-11-11T17:21:09.189770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519118519688298:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:09.189849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:09.201683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519109929753605:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(43599935ns) {HandleHandshakeStatus:329(43596828ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(422ns) SwitchToState:178(328ns) ProcessPendingSessionEvents:495(830ns)}}
2025-11-11T17:21:09.267230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:728: Handshake [6:7571519113905590338:2117] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-11T17:21:09.267315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:09.268483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519111957482217:2048] [node 1] ICS01 socket: 575 reason# EndOfStream
2025-11-11T17:21:09.268614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519107662514826:2048] [node 1] ICP30 unregister session Session# [2:7571519111957482217:2048] VirtualId# [2:7571519111957482205:0]
2025-11-11T17:21:09.268651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519111957482217:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.268738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:09.268789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:09.269886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519118778523930:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:09.269998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:09.270171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519114483556631:2048] [node 1] ICS01 socket: 603 reason# EndOfStream
2025-11-11T17:21:09.270205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519110188589240:2048] [node 1] ICP30 unregister session Session# [3:7571519114483556631:2048] VirtualId# [3:7571519114483556618:0]
2025-11-11T17:21:09.270238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519114483556631:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.270319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:09.270356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:09.273760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519114483556601:2107] [node 6] ICP18 handshake (actor [3:7571519118778523930:2121]) is too old
2025-11-11T17:21:09.281722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:09.282002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519118200557647:2048] [node 1] ICS07 socket disconnect 605 reason# EndOfStream
2025-11-11T17:21:09.282036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519118200557647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.282124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:09.282167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519118200557647:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:09.281818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:09.281818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:09.281990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519116966627337:2048] [node 1] ICS01 socket: 626 reason# EndOfStream
2025-11-11T17:21:09.282017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519108376692644:2048] [node 1] ICP30 unregister session Session# [5:7571519116966627337:2048] VirtualId# [5:7571519112671660038:0]
2025-11-11T17:21:09.282040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519116966627337:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.282120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:09.282157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:09.282754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519114224720991:2048] [node 1] ICS01 socket: 580 reason# EndOfStream
2025-11-11T17:21:09.282795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519109929753604:2048] [node 1] ICP30 unregister session Session# [4:7571519114224720991:2048] VirtualId# [4:7571519114224720979:0]
2025-11-11T17:21:09.282967Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Couldn't connect to any resolved address
2025-11-11T17:21:09.282822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519114224720991:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.289777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519109610622960:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:09.294035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519118200557647:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:09.294082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519109610622960:2048] [node 1] ICP30 unregister session Session# [6:7571519118200557647:2048] VirtualId# [6:7571519113905590343:0]
2025-11-11T17:21:09.294142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:09.294179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519109610622960:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:09.294193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12618) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:09.282897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:09.282953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:09.309859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519109610622960:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:09.341769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:09.342022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519114483556624:2049] [node 2] ICS07 socket disconnect 573 reason# EndOfStream
2025-11-11T17:21:09.342059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519114483556624:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.342140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:09.342178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519114483556624:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:09.348866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:09.349041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519118200557642:2049] [node 2] ICS01 socket: 577 reason# EndOfStream
2025-11-11T17:21:09.349067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519109610622961:2049] [node 2] ICP30 unregister session Session# [6:7571519118200557642:2049] VirtualId# [6:7571519113905590328:0]
2025-11-11T17:21:09.349089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519118200557642:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.349144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:09.349171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:09.345148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:09.349497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:09.345368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519118519688298:2049] [node 2] ICS01 socket: 610 reason# EndOfStream
2025-11-11T17:21:09.345412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519109929753605:2049] [node 2] ICP30 unregister session Session# [4:7571519118519688298:2049] VirtualId# [4:7571519114224720993:0]
2025-11-11T17:21:09.354016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519112671660036:2049] [node 2] ICS01 socket: 586 reason# EndOfStream
2025-11-11T17:21:09.359800Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:21:09.345728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519118519688298:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.357842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519108376692645:2049] [node 2] ICP30 unregister session Session# [5:7571519112671660036:2049] VirtualId# [5:7571519112671660025:0]
2025-11-11T17:21:09.345804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:09.345853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:09.362506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519110188589241:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:09.364194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519114483556624:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:09.357897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519112671660036:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.358014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:09.358062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:09.367983Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Couldn't connect to any resolved address
2025-11-11T17:21:09.368311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519110188589241:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:09.368363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519114483556624:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:09.368409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519110188589241:2049] [node 2] ICP30 unregister session Session# [3:7571519114483556624:2049] VirtualId# [3:7571519114483556611:0]
2025-11-11T17:21:09.368451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:09.368480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519110188589241:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:09.368493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7436) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:09.437784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:09.438015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519118200557653:2050] [node 3] ICS01 socket: 582 reason# EndOfStream
2025-11-11T17:21:09.438061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519109610622962:2050] [node 3] ICP30 unregister session Session# [6:7571519118200557653:2050] VirtualId# [6:7571519118200557645:0]
2025-11-11T17:21:09.438095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519118200557653:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.438203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:09.438250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:09.438342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:09.438538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519116966627339:2050] [node 3] ICS01 socket: 630 reason# EndOfStream
2025-11-11T17:21:09.438575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519108376692646:2050] [node 3] ICP30 unregister session Session# [5:7571519116966627339:2050] VirtualId# [5:7571519112671660035:0]
2025-11-11T17:21:09.438601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519116966627339:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.438667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:09.438701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:09.439969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:09.440272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519118519688292:2050] [node 3] ICS01 socket: 599 reason# EndOfStream
2025-11-11T17:21:09.440331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519109929753606:2050] [node 3] ICP30 unregister session Session# [4:7571519118519688292:2050] VirtualId# [4:7571519114224720987:0]
2025-11-11T17:21:09.440387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519118519688292:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.440513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:09.440575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:09.525795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:09.526034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519118200557655:2069] [node 4] ICS07 socket disconnect 622 reason# EndOfStream
2025-11-11T17:21:09.526071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519118200557655:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.526167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:09.526208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519118200557655:2069] [node 4] ICS15 start handshake
2025-11-11T17:21:09.526573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:09.526774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519112671660027:2086] [node 4] ICS01 socket: 583 reason# EndOfStream
2025-11-11T17:21:09.526811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519112671659995:2086] [node 4] ICP30 unregister session Session# [5:7571519112671660027:2086] VirtualId# [5:7571519112671660018:0]
2025-11-11T17:21:09.526843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519112671660027:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.526909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:09.526949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:09.527622Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Couldn't connect to any resolved address
2025-11-11T17:21:09.527739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519113905590291:2069] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:09.527849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519118200557655:2069] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:09.528029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519113905590291:2069] [node 4] ICP30 unregister session Session# [6:7571519118200557655:2069] VirtualId# [6:7571519118200557643:0]
2025-11-11T17:21:09.528076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:09.528108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519113905590291:2069] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:09.528122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:09.537936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519113905590291:2069] [node 4] ICP33 wake up from error state
2025-11-11T17:21:09.643968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:09.644392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519118200557646:2095] [node 5] ICS01 socket: 593 reason# EndOfStream
2025-11-11T17:21:09.644452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519113905590304:2095] [node 5] ICP30 unregister session Session# [6:7571519118200557646:2095] VirtualId# [6:7571519113905590333:0]
2025-11-11T17:21:09.644493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519118200557646:2095] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:09.644632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:09.644722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:09.765829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519115774667259:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:09.765886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519115774667260:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:09.784704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519116808810402:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:09.784773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519116808810403:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:09.814512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519117344708016:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:09.814570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519117344708017:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:09.829318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519118803584527:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:09.829357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519118803584528:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:09.829385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519118803584529:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:09.851923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519117700111133:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:09.851983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519117700111134:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:09.852025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519117700111135:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:09.873400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519116022595621:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:09.873446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519116022595622:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:09.873477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519116022595623:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:10.135322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519121995078443:2060] [node 6] ICP01 ready to work
2025-11-11T17:21:10.226033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519120069634618:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:10.218341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519121639675380:2107] [node 6] ICP01 ready to work
2025-11-11T17:21:10.242781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519121103777761:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:10.226423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519120069634619:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:10.307418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519121639675381:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:10.317975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519121639675382:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:10.226709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519120069634620:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:10.310294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519121103777762:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:10.342964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519118803584527:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(207531808ns) {Configure:104(207520606ns) {ProcessConfigured:135(62614ns) {StartInitialHandshake:157(59845ns) {DropHandshakes:469(2309ns) {DropIncomingHandshake:441(529ns) DropOutgoingHandshake:456(343ns)} GenerateSessionVirtualId:988(649ns) PrepareNewSessionHandshake:476(1515ns) {SwitchToState:178(599ns)}}}}}
2025-11-11T17:21:10.326877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519121103777763:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:10.368298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519121995078443:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(214097867ns) {Configure:104(214083222ns) {ProcessConfigured:135(73070ns) {StartInitialHandshake:157(70472ns) {DropHandshakes:469(2093ns) {DropIncomingHandshake:441(591ns) DropOutgoingHandshake:456(350ns)} GenerateSessionVirtualId:988(768ns) PrepareNewSessionHandshake:476(1501ns) {SwitchToState:178(559ns)}}}}}
2025-11-11T17:21:10.368858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519121995078495:2070] [node 4] ICP01 ready to work
2025-11-11T17:21:10.354766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519123098551892:2086] [node 5] ICP01 ready to work
2025-11-11T17:21:10.378033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519115774667260:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(146058100ns) {Configure:104(146052036ns) {ProcessConfigured:135(31749080ns) {StartInitialHandshake:157(31746545ns) {DropHandshakes:469(6988109ns) {DropIncomingHandshake:441(189ns) DropOutgoingHandshake:456(5530ns)} GenerateSessionVirtualId:988(4778ns) PrepareNewSessionHandshake:476(7425ns) {SwitchToState:178(2568ns)}}}}}
2025-11-11T17:21:10.355072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519123098551893:2088] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.381654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519123098551893:2088] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24294422ns) {Configure:104(24291225ns) {ProcessConfigured:135(47545ns) {StartInitialHandshake:157(44172ns) {DropHandshakes:469(1938ns) {DropIncomingHandshake:441(438ns) DropOutgoingHandshake:456(362ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:476(1263ns) {SwitchToState:178(606ns)}}}}}
2025-11-11T17:21:10.393887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078504:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.398242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519116022595621:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(144760712ns) {Configure:104(144755402ns) {ProcessConfigured:135(57333ns) {StartInitialHandshake:157(54780ns) {DropHandshakes:469(2343ns) {DropIncomingHandshake:441(490ns) DropOutgoingHandshake:456(473ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:476(4961ns) {SwitchToState:178(592ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.422719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519116022595622:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24223620ns) {Configure:104(24218602ns) {ProcessConfigured:135(85024ns) {StartInitialHandshake:157(81405ns) {DropHandshakes:469(3581ns) {DropIncomingHandshake:441(686ns) DropOutgoingHandshake:456(501ns)} GenerateSessionVirtualId:988(858ns) PrepareNewSessionHandshake:476(1825ns) {SwitchToState:178(539ns)}}}}}
2025-11-11T17:21:10.415962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519120069634618:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20300344ns) {Configure:104(20294272ns) {ProcessConfigured:135(7331863ns) {StartInitialHandshake:157(6983317ns) {DropHandshakes:469(3018ns) {DropIncomingHandshake:441(720ns) DropOutgoingHandshake:456(513ns)} GenerateSessionVirtualId:988(1153ns) PrepareNewSessionHandshake:476(3566ns) {SwitchToState:178(1315ns)}}}}}
2025-11-11T17:21:10.442447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519116808810402:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(102143997ns) {Configure:104(100684541ns) {ProcessConfigured:135(67693ns) {StartInitialHandshake:157(64411ns) {DropHandshakes:469(3031ns) {DropIncomingHandshake:441(594ns) DropOutgoingHandshake:456(498ns)} GenerateSessionVirtualId:988(1343ns) PrepareNewSessionHandshake:476(1480ns) {SwitchToState:178(438ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.423163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519120317562984:2064] [node 5] ICP01 ready to work
2025-11-11T17:21:10.430965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519120069634631:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17745
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.454402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551901:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.431025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519120069634632:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17745
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.423529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519120317562985:2069] [node 4] ICP01 ready to work
2025-11-11T17:21:10.473739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519121103777761:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16875275ns) {Configure:104(16871040ns) {ProcessConfigured:135(70016ns) {StartInitialHandshake:157(67247ns) {DropHandshakes:469(3095ns) {DropIncomingHandshake:441(661ns) DropOutgoingHandshake:456(677ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:476(1622ns) {SwitchToState:178(639ns)}}}}}
2025-11-11T17:21:10.443979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519120317562984:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19645189ns) {Configure:104(19639763ns) {ProcessConfigured:135(5533641ns) {StartInitialHandshake:157(5530882ns) {DropHandshakes:469(2766ns) {DropIncomingHandshake:441(938ns) DropOutgoingHandshake:456(383ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:476(4059ns) {SwitchToState:178(1664ns)}}}}}
2025-11-11T17:21:10.447541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317562986:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.447724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317562987:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.501282Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519120317562987:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.501509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519120317562984:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.431079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519120069634633:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17745
2025-11-11T17:21:10.485865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519123098551901:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.468695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634632:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.469132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519120069634637:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17745
2025-11-11T17:21:10.497733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634637:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.510195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519121103777763:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(34545961ns) {Configure:104(34540359ns) {ProcessConfigured:135(4111191ns) {StartInitialHandshake:157(4107158ns) {DropHandshakes:469(3100ns) {DropIncomingHandshake:441(646ns) DropOutgoingHandshake:456(923ns)} GenerateSessionVirtualId:988(1410ns) PrepareNewSessionHandshake:476(5138ns) {SwitchToState:178(2385ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.494108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519118803584527:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.527543Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634626:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.527593Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519120069634626:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.528344Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634633:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.529834Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519120069634626:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:10.529416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317562998:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.538138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551903:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.525753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519121995078494:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.542980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519121639675396:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62221
2025-11-11T17:21:10.543122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519121639675397:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62221
2025-11-11T17:21:10.543237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519121639675398:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62221
2025-11-11T17:21:10.543684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519120069634618:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.525818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519121995078494:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.526755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078508:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.541975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519121995078494:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:10.543765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:10.543817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519120069634618:2107] [node 4] ICP22 created new session: [1:7571519120069634642:2107]
2025-11-11T17:21:10.543874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519120069634642:2107] [node 4] ICS09 handshake done sender: [1:7571519120069634626:2112] self: [1:7571519120069634625:0] peer: [4:7571519123098551902:0] socket: 595 qp: -1
2025-11-11T17:21:10.543911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519120069634642:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:10.543952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:10.544628Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634631:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.547814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519121103777774:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8775
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.547974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519121103777775:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8775
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.545533Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519123098551901:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:10.546239Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675398:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.548779Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519121995078508:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.550746Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634624:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.550787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519120069634624:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.548075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519121103777776:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8775
2025-11-11T17:21:10.553869Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519120317562986:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.554129Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519120317562987:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:10.548877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519121995078504:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.556166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777775:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.556305Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777776:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.560129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519120317562984:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.560191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:10.560245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519120317562984:2064] [node 5] ICP22 created new session: [6:7571519120317562999:2064]
2025-11-11T17:21:10.560289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519120317562999:2064] [node 5] ICS09 handshake done sender: [6:7571519120317562987:2113] self: [6:7571519120317562997:0] peer: [5:7571519121995078493:0] socket: 589 qp: -1
2025-11-11T17:21:10.560322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519120317562999:2064] [node 5] ICS10 traffic start
2025-11-11T17:21:10.560357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:10.560914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519116022595623:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.549169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519121995078443:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.546338Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675396:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.546851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519117344708016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.557975Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675397:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.586688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519120069634646:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17745
2025-11-11T17:21:10.549212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:10.558412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519121639675401:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62221
2025-11-11T17:21:10.558469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519121639675402:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62221
2025-11-11T17:21:10.593859Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:10.586953Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519120069634624:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:10.604079Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634628:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.604115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519120069634628:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.604724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519115774667260:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.606830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519116808810402:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.607156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519121103777780:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8775
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.549268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519121995078443:2060] [node 6] ICP22 created new session: [5:7571519121995078511:2060]
2025-11-11T17:21:10.610078Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777780:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.618264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317563003:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.618342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317563004:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.618900Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519120317562986:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:10.619140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519116022595623:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.619197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:10.619249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519116022595623:2050] [node 3] ICP22 created new session: [6:7571519120317563005:2050]
2025-11-11T17:21:10.619298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519120317563005:2050] [node 3] ICS09 handshake done sender: [6:7571519120317562986:2112] self: [6:7571519120317563001:0] peer: [3:7571519121639675383:0] socket: 588 qp: -1
2025-11-11T17:21:10.619337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519120317563005:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:10.619367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:10.598798Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675388:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.604788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:10.549311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519121995078511:2060] [node 6] ICS09 handshake done sender: [5:7571519121995078494:2110] self: [5:7571519121995078493:0] peer: [6:7571519120317562997:0] socket: 583 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.604843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519115774667260:2049] [node 3] ICP22 created new session: [1:7571519120069634648:2049]
2025-11-11T17:21:10.549345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519121995078511:2060] [node 6] ICS10 traffic start
2025-11-11T17:21:10.598861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519121639675388:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.629876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519118803584527:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.630007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:10.630063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519118803584527:2048] [node 1] ICP22 created new session: [4:7571519123098551906:2048]
2025-11-11T17:21:10.630116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519123098551906:2048] [node 1] ICS09 handshake done sender: [4:7571519123098551901:2115] self: [4:7571519123098551902:0] peer: [1:7571519120069634625:0] socket: 599 qp: -1
2025-11-11T17:21:10.630158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519123098551906:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:10.630194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:10.630464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519118803584527:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:10.631341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551904:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.631496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551905:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.604888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519120069634648:2049] [node 3] ICS09 handshake done sender: [1:7571519120069634624:2111] self: [1:7571519120069634623:0] peer: [3:7571519121639675399:0] socket: 594 qp: -1
2025-11-11T17:21:10.603054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675384:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.654605Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777774:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.654864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519121103777783:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8775
2025-11-11T17:21:10.549379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:10.604924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519120069634648:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:10.604969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:10.607954Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634646:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.608055Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634622:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.608074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519120069634622:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.622161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519120069634628:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:10.655752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519120069634622:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:10.656288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519120069634619:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.656347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:10.656386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519120069634619:2108] [node 5] ICP22 created new session: [1:7571519120069634654:2108]
2025-11-11T17:21:10.656427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519120069634654:2108] [node 5] ICS09 handshake done sender: [1:7571519120069634628:2113] self: [1:7571519120069634627:0] peer: [5:7571519121995078515:0] socket: 591 qp: -1
2025-11-11T17:21:10.656455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519120069634654:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:10.656483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:10.656878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519115774667259:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.656922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:10.656969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519115774667259:2048] [node 2] ICP22 created new session: [1:7571519120069634656:2048]
2025-11-11T17:21:10.657011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519120069634656:2048] [node 2] ICS09 handshake done sender: [1:7571519120069634622:2110] self: [1:7571519120069634621:0] peer: [2:7571519121103777781:0] socket: 593 qp: -1
2025-11-11T17:21:10.657042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519120069634656:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:10.657109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:10.660076Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519121103777775:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:10.603082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519121639675384:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.610574Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:10.611648Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519117344708016:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519121639675398:2122] held: no
2025-11-11T17:21:10.612027Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519121639675398:2122] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:10.612399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519121639675404:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62221
2025-11-11T17:21:10.612581Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519121639675388:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:10.612912Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519121639675384:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:10.613201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519117344708016:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.622987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:10.645770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519117344708016:2048] [node 1] ICP22 created new session: [3:7571519121639675407:2048]
2025-11-11T17:21:10.650992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519121639675407:2048] [node 1] ICS09 handshake done sender: [3:7571519121639675398:2122] self: [3:7571519121639675399:0] peer: [1:7571519120069634623:0] socket: 606 qp: -1
2025-11-11T17:21:10.651047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519121639675407:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:10.651098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:10.651374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519117344708016:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(38066771ns) {HandleHandshakeStatus:329(38064048ns) {DropIncomingHandshake:441(826ns) DropOutgoingHandshake:456(354ns) SwitchToState:178(884ns) ProcessPendingSessionEvents:495(1276ns)}}
2025-11-11T17:21:10.651423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519121639675381:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.651530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:10.651577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519121639675381:2108] [node 5] ICP22 created new session: [3:7571519121639675409:2108]
2025-11-11T17:21:10.651626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519121639675409:2108] [node 5] ICS09 handshake done sender: [3:7571519121639675388:2117] self: [3:7571519121639675387:0] peer: [5:7571519121995078514:0] socket: 578 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.651654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519121639675409:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:10.661037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519116808810402:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.661091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:10.661131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519116808810402:2048] [node 1] ICP22 created new session: [2:7571519121103777787:2048]
2025-11-11T17:21:10.661177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519121103777787:2048] [node 1] ICS09 handshake done sender: [2:7571519121103777775:2116] self: [2:7571519121103777781:0] peer: [1:7571519120069634621:0] socket: 603 qp: -1
2025-11-11T17:21:10.661210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519121103777787:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:10.661267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:10.661884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519121103777786:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8775
2025-11-11T17:21:10.651691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:10.652053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519121639675380:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.652115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:10.652153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519121639675380:2107] [node 6] ICP22 created new session: [3:7571519121639675411:2107]
2025-11-11T17:21:10.652193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519121639675411:2107] [node 6] ICS09 handshake done sender: [3:7571519121639675384:2115] self: [3:7571519121639675383:0] peer: [6:7571519120317563001:0] socket: 576 qp: -1
2025-11-11T17:21:10.652228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519121639675411:2107] [node 6] ICS10 traffic start
2025-11-11T17:21:10.652253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:10.652825Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675404:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.658742Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675401:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.659072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519121639675380:2107] [node 6] ICP18 handshake (actor [3:7571519121639675404:2125]) is too old
2025-11-11T17:21:10.665277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519117344708017:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.559276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571519121995078443:2060] [node 6] ICP18 handshake (actor [5:7571519121995078508:2116]) is too old
2025-11-11T17:21:10.559375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078510:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.559557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519117700111135:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.585862Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519121995078510:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.587611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519117700111133:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.668805Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.669604Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519123098551905:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.669740Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519123098551904:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.670240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519118803584528:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.670312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551909:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.671005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519123098551909:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.671101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519118803584529:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.671387Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519121639675392:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.671416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519121639675392:2119] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.672529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551913:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.672741Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519121639675392:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:10.673233Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519123098551909:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:10.674223Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777767:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.674255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519121103777767:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.674859Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777769:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.674878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519121103777769:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.676323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519121639675382:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.676417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:10.676463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519121639675382:2109] [node 4] ICP22 created new session: [3:7571519121639675417:2109]
2025-11-11T17:21:10.676510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519121639675417:2109] [node 4] ICS09 handshake done sender: [3:7571519121639675392:2119] self: [3:7571519121639675391:0] peer: [4:7571519123098551912:0] socket: 630 qp: -1
2025-11-11T17:21:10.676551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519121639675417:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:10.676581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:10.673452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519118803584529:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.680430Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519121103777767:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:10.680864Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519120317563003:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.681007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519116022595621:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.673500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.678150Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:10.673543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519118803584529:2050] [node 3] ICP22 created new session: [4:7571519123098551914:2050]
2025-11-11T17:21:10.673629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519123098551914:2050] [node 3] ICS09 handshake done sender: [4:7571519123098551909:2119] self: [4:7571519123098551912:0] peer: [3:7571519121639675391:0] socket: 592 qp: -1
2025-11-11T17:21:10.673668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519123098551914:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:10.673706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:10.683219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078516:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.681758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519121639675419:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62221
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.685012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317563009:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.685460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078517:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.681984Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519120069634630:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.682020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519120069634630:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.685930Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519120069634630:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:10.686612Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519120317563003:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:10.686973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519120069634620:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.687040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:10.687083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519120069634620:2109] [node 6] ICP22 created new session: [1:7571519120069634659:2109]
2025-11-11T17:21:10.687134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519120069634659:2109] [node 6] ICS09 handshake done sender: [1:7571519120069634630:2114] self: [1:7571519120069634629:0] peer: [6:7571519120317563008:0] socket: 619 qp: -1
2025-11-11T17:21:10.687169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519120069634659:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:10.687206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:10.688266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519116022595621:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.688320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:10.688364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519116022595621:2048] [node 1] ICP22 created new session: [6:7571519120317563010:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.690890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551916:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.697977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519121103777769:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:10.688402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519120317563010:2048] [node 1] ICS09 handshake done sender: [6:7571519120317563003:2118] self: [6:7571519120317563008:0] peer: [1:7571519120069634629:0] socket: 621 qp: -1
2025-11-11T17:21:10.692946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519117700111135:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519121995078504:2115] held: no
2025-11-11T17:21:10.693205Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519117700111133:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519121995078510:2117] held: no
2025-11-11T17:21:10.693382Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519121995078504:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:10.698151Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519121639675401:2123] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:10.688444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519120317563010:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:10.688478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.702933Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519121995078510:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:10.708851Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519123098551904:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:10.706598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519117344708017:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.708267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078518:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.709068Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777786:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.706715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:10.706764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519117344708017:2049] [node 2] ICP22 created new session: [3:7571519121639675420:2049]
2025-11-11T17:21:10.706821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519121639675420:2049] [node 2] ICS09 handshake done sender: [3:7571519121639675401:2123] self: [3:7571519121639675415:0] peer: [2:7571519121103777766:0] socket: 614 qp: -1
2025-11-11T17:21:10.706861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519121639675420:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:10.706894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:10.709210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519116808810403:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.715187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317563012:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.708416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519117700111135:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.709270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:10.715462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519118803584528:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.708671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:10.709310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519116808810403:2049] [node 3] ICP22 created new session: [2:7571519121103777792:2049]
2025-11-11T17:21:10.713724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519117700111135:2050] [node 3] ICP22 created new session: [5:7571519121995078520:2050]
2025-11-11T17:21:10.709346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519121103777792:2049] [node 3] ICS09 handshake done sender: [2:7571519121103777767:2111] self: [2:7571519121103777766:0] peer: [3:7571519121639675415:0] socket: 611 qp: -1
2025-11-11T17:21:10.709378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519121103777792:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:10.709416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:10.709908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519121103777761:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.709954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:10.710002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519121103777761:2107] [node 4] ICP22 created new session: [2:7571519121103777794:2107]
2025-11-11T17:21:10.710042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519121103777794:2107] [node 4] ICS09 handshake done sender: [2:7571519121103777769:2112] self: [2:7571519121103777768:0] peer: [4:7571519123098551911:0] socket: 612 qp: -1
2025-11-11T17:21:10.710065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519121103777794:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:10.710134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:10.715846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:10.714959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519121995078520:2050] [node 3] ICS09 handshake done sender: [5:7571519121995078504:2115] self: [5:7571519121995078514:0] peer: [3:7571519121639675387:0] socket: 582 qp: -1
2025-11-11T17:21:10.715002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519121995078520:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:10.715544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:10.716207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519117700111133:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.716239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:10.716275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519117700111133:2048] [node 1] ICP22 created new session: [5:7571519121995078523:2048]
2025-11-11T17:21:10.716316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519121995078523:2048] [node 1] ICS09 handshake done sender: [5:7571519121995078510:2117] self: [5:7571519121995078515:0] peer: [1:7571519120069634627:0] socket: 613 qp: -1
2025-11-11T17:21:10.716361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519121995078523:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:10.716430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:10.717008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078519:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.719989Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519121995078518:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.727573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519121995078495:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.717504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519118803584528:2049] [node 2] ICP22 created new session: [4:7571519123098551918:2049]
2025-11-11T17:21:10.717556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519123098551918:2049] [node 2] ICS09 handshake done sender: [4:7571519123098551904:2117] self: [4:7571519123098551911:0] peer: [2:7571519121103777768:0] socket: 627 qp: -1
2025-11-11T17:21:10.717602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519123098551918:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:10.717634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:10.727660Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18210) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:10.731158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519118803584528:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:10.731574Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519123098551895:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.731601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519123098551895:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.732448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519123098551920:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61455
2025-11-11T17:21:10.732749Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519120317563012:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.732890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519120317562985:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.733578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519116808810403:2049] [node 3] ICP18 handshake (actor [2:7571519121103777786:2120]) is too old
2025-11-11T17:21:10.734852Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519123098551895:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:10.735321Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519123098551897:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.735348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519123098551897:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.737190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317563014:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.735986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519123098551892:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.736020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:10.736066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519123098551892:2086] [node 5] ICP22 created new session: [4:7571519123098551922:2086]
2025-11-11T17:21:10.736115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519123098551922:2086] [node 5] ICS09 handshake done sender: [4:7571519123098551895:2113] self: [4:7571519123098551894:0] peer: [5:7571519121995078525:0] socket: 629 qp: -1
2025-11-11T17:21:10.736147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519123098551922:2086] [node 5] ICS10 traffic start
2025-11-11T17:21:10.736175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:10.737371Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519123098551897:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:10.742057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519120317563012:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:10.742308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519120317562985:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.742359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:10.742407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519120317562985:2069] [node 4] ICP22 created new session: [6:7571519120317563015:2069]
2025-11-11T17:21:10.744236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519123098551893:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.745950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519120317563015:2069] [node 4] ICS09 handshake done sender: [6:7571519120317563012:2121] self: [6:7571519120317563013:0] peer: [4:7571519123098551896:0] socket: 615 qp: -1
2025-11-11T17:21:10.745997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519120317563015:2069] [node 4] ICS10 traffic start
2025-11-11T17:21:10.746031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:10.748848Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:10.744314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:10.744359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519123098551893:2088] [node 6] ICP22 created new session: [4:7571519123098551925:2088]
2025-11-11T17:21:10.744408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519123098551925:2088] [node 6] ICS09 handshake done sender: [4:7571519123098551897:2114] self: [4:7571519123098551896:0] peer: [6:7571519120317563013:0] socket: 596 qp: -1
2025-11-11T17:21:10.744447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519123098551925:2088] [node 6] ICS10 traffic start
2025-11-11T17:21:10.744482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:10.745002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519123098551920:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.748159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519123098551892:2086] [node 5] ICP18 handshake (actor [4:7571519123098551920:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.751732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519121995078495:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519121995078518:2120] held: no
2025-11-11T17:21:10.751942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519121995078519:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.752034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078526:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.752981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317563017:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.753350Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519121995078518:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:10.753675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519117700111134:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:10.753728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519121995078495:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.753794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:10.753845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519121995078495:2070] [node 4] ICP22 created new session: [5:7571519121995078528:2070]
2025-11-11T17:21:10.753887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519121995078528:2070] [node 4] ICS09 handshake done sender: [5:7571519121995078518:2120] self: [5:7571519121995078525:0] peer: [4:7571519123098551894:0] socket: 579 qp: -1
2025-11-11T17:21:10.753920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519121995078528:2070] [node 4] ICS10 traffic start
2025-11-11T17:21:10.753953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:10.754902Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777771:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.754937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519121103777771:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.755731Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519120317563017:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.755874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519116022595622:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.757204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519121995078530:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30073
2025-11-11T17:21:10.757473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519121103777773:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:10.757500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519121103777773:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:10.758184Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519121103777771:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:10.758707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519121103777762:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.758762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:10.758802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519121103777762:2108] [node 5] ICP22 created new session: [2:7571519121103777799:2108]
2025-11-11T17:21:10.758840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519121103777799:2108] [node 5] ICS09 handshake done sender: [2:7571519121103777771:2113] self: [2:7571519121103777770:0] peer: [5:7571519121995078527:0] socket: 634 qp: -1
2025-11-11T17:21:10.758870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519121103777799:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:10.758937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:10.760063Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519121995078519:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:10.760317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519117700111134:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.760404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:10.760449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519117700111134:2049] [node 2] ICP22 created new session: [5:7571519121995078531:2049]
2025-11-11T17:21:10.760489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519121995078531:2049] [node 2] ICS09 handshake done sender: [5:7571519121995078519:2121] self: [5:7571519121995078527:0] peer: [2:7571519121103777770:0] socket: 600 qp: -1
2025-11-11T17:21:10.760521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519121995078531:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:10.760558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:10.762335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519120317563019:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36616
2025-11-11T17:21:10.763102Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519121103777773:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:10.763611Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519120317563017:2123] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:10.763848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519121103777763:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:10.763928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:10.763971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519121103777763:2109] [node 6] ICP22 created new session: [2:7571519121103777801:2109]
2025-11-11T17:21:10.764017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519121103777801:2109] [node 6] ICS09 handshake done sender: [2:7571519121103777773:2114] self: [2:7571519121103777772:0] peer: [6:7571519120317563018:0] socket: 638 qp: -1
2025-11-11T17:21:10.764046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519121103777801:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:10.764081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:10.764867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519116022595622:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:10.764946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:10.764992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519116022595622:2049] [node 2] ICP22 created new session: [6:7571519120317563020:2049]
2025-11-11T17:21:10.765038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519120317563020:2049] [node 2] ICS09 handshake done sender: [6:7571519120317563017:2123] self: [6:7571519120317563018:0] peer: [2:7571519121103777772:0] socket: 581 qp: -1
2025-11-11T17:21:10.765077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519120317563020:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:10.765104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:10.788025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:10.788241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519121103777787:2048] [node 1] ICS01 socket: 603 reason# EndOfStream
2025-11-11T17:21:10.788291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519116808810402:2048] [node 1] ICP30 unregister session Session# [2:7571519121103777787:2048] VirtualId# [2:7571519121103777781:0]
2025-11-11T17:21:10.788329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519121103777787:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.788411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:10.788459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:10.793751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:10.794676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519121639675407:2048] [node 1] ICS01 socket: 606 reason# EndOfStream
2025-11-11T17:21:10.794726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519117344708016:2048] [node 1] ICP30 unregister session Session# [3:7571519121639675407:2048] VirtualId# [3:7571519121639675399:0]
2025-11-11T17:21:10.794765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519121639675407:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.794846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:10.794886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:10.797763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:10.798003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519123098551906:2048] [node 1] ICS01 socket: 599 reason# EndOfStream
2025-11-11T17:21:10.798050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519118803584527:2048] [node 1] ICP30 unregister session Session# [4:7571519123098551906:2048] VirtualId# [4:7571519123098551902:0]
2025-11-11T17:21:10.798088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519123098551906:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.798160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:10.798206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:10.801730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:10.801971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519121995078523:2048] [node 1] ICS01 socket: 613 reason# EndOfStream
2025-11-11T17:21:10.802014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519117700111133:2048] [node 1] ICP30 unregister session Session# [5:7571519121995078523:2048] VirtualId# [5:7571519121995078515:0]
2025-11-11T17:21:10.802047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519121995078523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.802138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:10.802186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:10.805740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:10.805962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519120317563010:2048] [node 1] ICS01 socket: 621 reason# EndOfStream
2025-11-11T17:21:10.806006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519116022595621:2048] [node 1] ICP30 unregister session Session# [6:7571519120317563010:2048] VirtualId# [6:7571519120317563008:0]
2025-11-11T17:21:10.806044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519120317563010:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.806138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:10.806182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:10.853837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:10.854076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519121639675420:2049] [node 2] ICS01 socket: 614 reason# EndOfStream
2025-11-11T17:21:10.854128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519117344708017:2049] [node 2] ICP30 unregister session Session# [3:7571519121639675420:2049] VirtualId# [3:7571519121639675415:0]
2025-11-11T17:21:10.854168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519121639675420:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.854270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:10.854337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:10.861844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:10.861915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:10.862144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519120317563020:2049] [node 2] ICS01 socket: 581 reason# EndOfStream
2025-11-11T17:21:10.862195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519116022595622:2049] [node 2] ICP30 unregister session Session# [6:7571519120317563020:2049] VirtualId# [6:7571519120317563018:0]
2025-11-11T17:21:10.862230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519120317563020:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.862315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:10.862357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:10.862214Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:10.862400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519121995078531:2049] [node 2] ICS01 socket: 600 reason# EndOfStream
2025-11-11T17:21:10.862443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519117700111134:2049] [node 2] ICP30 unregister session Session# [5:7571519121995078531:2049] VirtualId# [5:7571519121995078527:0]
2025-11-11T17:21:10.862469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519121995078531:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.862545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:10.862598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:10.862820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519123098551918:2049] [node 2] ICS01 socket: 627 reason# EndOfStream
2025-11-11T17:21:10.862875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519118803584528:2049] [node 2] ICP30 unregister session Session# [4:7571519123098551918:2049] VirtualId# [4:7571519123098551911:0]
2025-11-11T17:21:10.862902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519123098551918:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.862965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:10.863003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:10.894143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:10.894425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519121995078520:2050] [node 3] ICS01 socket: 582 reason# EndOfStream
2025-11-11T17:21:10.894470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519117700111135:2050] [node 3] ICP30 unregister session Session# [5:7571519121995078520:2050] VirtualId# [5:7571519121995078514:0]
2025-11-11T17:21:10.894512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519121995078520:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.894590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:10.894629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:10.895544Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:10.897703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:10.895745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519120317563005:2050] [node 3] ICS01 socket: 588 reason# EndOfStream
2025-11-11T17:21:10.897914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519123098551914:2050] [node 3] ICS01 socket: 592 reason# EndOfStream
2025-11-11T17:21:10.895784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519116022595623:2050] [node 3] ICP30 unregister session Session# [6:7571519120317563005:2050] VirtualId# [6:7571519120317563001:0]
2025-11-11T17:21:10.897955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519118803584529:2050] [node 3] ICP30 unregister session Session# [4:7571519123098551914:2050] VirtualId# [4:7571519123098551912:0]
2025-11-11T17:21:10.895814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519120317563005:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.897988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519123098551914:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.895898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:10.898082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:10.895950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:10.898123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:10.965200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:10.965441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519121995078528:2070] [node 4] ICS01 socket: 579 reason# EndOfStream
2025-11-11T17:21:10.965494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519121995078495:2070] [node 4] ICP30 unregister session Session# [5:7571519121995078528:2070] VirtualId# [5:7571519121995078525:0]
2025-11-11T17:21:10.965527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519121995078528:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.965629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:10.965677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:10.969790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:10.970104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519120317563015:2069] [node 4] ICS01 socket: 615 reason# EndOfStream
2025-11-11T17:21:10.970160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519120317562985:2069] [node 4] ICP30 unregister session Session# [6:7571519120317563015:2069] VirtualId# [6:7571519120317563013:0]
2025-11-11T17:21:10.970200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519120317563015:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.970322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:10.970370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:10.995812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:10.996049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519120317562999:2064] [node 5] ICS01 socket: 589 reason# EndOfStream
2025-11-11T17:21:10.996094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519120317562984:2064] [node 5] ICP30 unregister session Session# [6:7571519120317562999:2064] VirtualId# [6:7571519120317562997:0]
2025-11-11T17:21:10.996132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519120317562999:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:10.996237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:10.996283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:11.061471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519126017863168:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:11.061512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519126017863169:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:11.095901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519125698020290:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:11.095969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519125698020291:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:11.103787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519127878967761:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:11.103851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519127878967762:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:11.140136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519126101421961:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:11.140209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519126101421962:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:11.140238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519126101421963:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:11.161646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519127857984734:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:11.161688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519127857984735:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:11.161716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519127857984736:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:11.293506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519124416019284:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:11.293567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519124416019285:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:11.293626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519124416019286:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:11.638110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519124416019329:2091] [node 5] ICP01 ready to work
2025-11-11T17:21:11.683574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519126017863231:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:11.683875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519126017863232:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:11.799245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519126017863233:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:11.858330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519127857984801:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:11.896622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519125698020355:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:11.925468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519126101421961:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(204336647ns) {Configure:104(204331328ns) {ProcessConfigured:135(70466ns) {StartInitialHandshake:157(67497ns) {DropHandshakes:469(3522ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(416ns)} GenerateSessionVirtualId:988(722ns) PrepareNewSessionHandshake:476(1574ns) {SwitchToState:178(697ns)}}}}}
2025-11-11T17:21:11.925931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519126101422027:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:11.922306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519124416019350:2096] [node 4] ICP01 ready to work
2025-11-11T17:21:11.858530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519127857984802:2111] [node 6] ICP01 ready to work
2025-11-11T17:21:11.938212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519126101422024:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:11.931542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519125698020356:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:11.948908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519124416019351:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:11.858645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519127857984795:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:11.965122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519125698020357:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:11.954222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519126017863169:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(270044001ns) {Configure:104(270039205ns) {ProcessConfigured:135(67664ns) {StartInitialHandshake:157(64970ns) {DropHandshakes:469(2745ns) {DropIncomingHandshake:441(688ns) DropOutgoingHandshake:456(616ns)} GenerateSessionVirtualId:988(708ns) PrepareNewSessionHandshake:476(1594ns) {SwitchToState:178(573ns)}}}}}
2025-11-11T17:21:11.909856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519127857984735:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(60059940ns) {Configure:104(60053133ns) {ProcessConfigured:135(13054318ns) {StartInitialHandshake:157(13050879ns) {DropHandshakes:469(2414ns) {DropIncomingHandshake:441(393ns) DropOutgoingHandshake:456(527ns)} GenerateSessionVirtualId:988(912ns) PrepareNewSessionHandshake:476(9223ns) {SwitchToState:178(4458ns)}}}}}
2025-11-11T17:21:11.965868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519127878967761:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(273701360ns) {Configure:104(273695760ns) {ProcessConfigured:135(33674152ns) {StartInitialHandshake:157(33670870ns) {DropHandshakes:469(5898684ns) {DropIncomingHandshake:441(1919ns) DropOutgoingHandshake:456(251ns)} GenerateSessionVirtualId:988(1752ns) PrepareNewSessionHandshake:476(3077ns) {SwitchToState:178(770ns)}}}}}
2025-11-11T17:21:11.966352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519127878967827:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:11.966636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519127878967828:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:11.966887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519127878967829:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:11.913942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519127857984801:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28065543ns) {Configure:104(28057830ns) {ProcessConfigured:135(13434138ns) {StartInitialHandshake:157(13430548ns) {DropHandshakes:469(2887ns) {DropIncomingHandshake:441(682ns) DropOutgoingHandshake:456(469ns)} GenerateSessionVirtualId:988(1164ns) PrepareNewSessionHandshake:476(2515ns) {SwitchToState:178(692ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:11.993927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519127878967829:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23243902ns) {Configure:104(23240746ns) {ProcessConfigured:135(16033869ns) {StartInitialHandshake:157(16031308ns) {DropHandshakes:469(2643ns) {DropIncomingHandshake:441(575ns) DropOutgoingHandshake:456(403ns)} GenerateSessionVirtualId:988(892ns) PrepareNewSessionHandshake:476(4301ns) {SwitchToState:178(1899ns)}}}}}
2025-11-11T17:21:11.994086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519126017863233:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(54016506ns) {Configure:104(54011298ns) {ProcessConfigured:135(37136008ns) {StartInitialHandshake:157(37132079ns) {DropHandshakes:469(2010ns) {DropIncomingHandshake:441(519ns) DropOutgoingHandshake:456(289ns)} GenerateSessionVirtualId:988(997ns) PrepareNewSessionHandshake:476(3364ns) {SwitchToState:178(1461ns)}}}}}
2025-11-11T17:21:11.994241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519124416019351:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.017827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519124416019286:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(245097087ns) {Configure:104(245087599ns) {ProcessConfigured:135(16949925ns) {StartInitialHandshake:157(16946971ns) {DropHandshakes:469(2547ns) {DropIncomingHandshake:441(550ns) DropOutgoingHandshake:456(483ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:476(4929ns) {SwitchToState:178(1815ns)}}}}}
2025-11-11T17:21:12.023554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519124416019329:2091] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.042707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519126101422037:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.042886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519130396389336:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
2025-11-11T17:21:11.989051Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519127857984795:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.061675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519126101422037:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.050084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132173935136:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
No RDMA context found for GID: 2025-11-11T17:21:12.019029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519126017863234:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38229
::ffff:127.1.0.0
2025-11-11T17:21:12.040963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952111:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:12.075117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519125698020291:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(77025625ns) {Configure:104(77016558ns) {ProcessConfigured:135(67985ns) {StartInitialHandshake:157(64478ns) {DropHandshakes:469(2871ns) {DropIncomingHandshake:441(542ns) DropOutgoingHandshake:456(446ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:476(1533ns) {SwitchToState:178(464ns)}}}}}
2025-11-11T17:21:12.073432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519127878967762:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(318529852ns) {Configure:104(318516875ns) {ProcessConfigured:135(46328321ns) {StartInitialHandshake:157(46326295ns) {DropHandshakes:469(3764ns) {DropIncomingHandshake:441(992ns) DropOutgoingHandshake:456(470ns)} GenerateSessionVirtualId:988(651ns) PrepareNewSessionHandshake:476(2741ns) {SwitchToState:178(1114ns)}}}}}
2025-11-11T17:21:12.019349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519130312830539:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38229
2025-11-11T17:21:12.019466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519130312830540:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38229
2025-11-11T17:21:12.019519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519130312830541:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38229
2025-11-11T17:21:12.029046Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519126017863234:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.030656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519126017863232:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(73245700ns) {Configure:104(73240952ns) {ProcessConfigured:135(67877ns) {StartInitialHandshake:157(65011ns) {DropHandshakes:469(2513ns) {DropIncomingHandshake:441(665ns) DropOutgoingHandshake:456(433ns)} GenerateSessionVirtualId:988(780ns) PrepareNewSessionHandshake:476(2201ns) {SwitchToState:178(924ns)}}}}}
2025-11-11T17:21:12.049805Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519130312830539:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.089804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519130312830540:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.070296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952110:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.114321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519129992987656:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-11T17:21:12.114458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519129992987657:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-11T17:21:12.114549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519129992987658:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.087109Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519127857984807:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.083625Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132173935136:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.125696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987657:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.125914Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987658:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.126045Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987656:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.119250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519130312830550:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38229
2025-11-11T17:21:12.138081Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519130312830541:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.116497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986661:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.143509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519125698020290:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.097693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132173935137:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-11T17:21:12.147403Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519130396389336:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.119380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986660:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.138399Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519124416019351:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:12.138687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519124416019329:2091] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.139151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:12.156485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519124416019329:2091] [node 5] ICP22 created new session: [6:7571519128710986662:2091]
2025-11-11T17:21:12.156552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519128710986662:2091] [node 5] ICS09 handshake done sender: [6:7571519124416019351:2111] self: [6:7571519128710986659:0] peer: [5:7571519127857984806:0] socket: 588 qp: -1
2025-11-11T17:21:12.156598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519128710986662:2091] [node 5] ICS10 traffic start
2025-11-11T17:21:12.156649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:12.157150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519124416019329:2091] [node 5] ICP35 event processing took too much time PendingConnection:278(18239424ns) {HandleHandshakeStatus:329(18237528ns) {DropIncomingHandshake:441(852ns) DropOutgoingHandshake:456(5930ns) SwitchToState:178(195ns) ProcessPendingSessionEvents:495(117116ns) {ForwardSessionEventToSession:612(64509ns) ForwardSessionEventToSession:612(3877ns) ForwardSessionEventToSession:612(3204ns) ForwardSessionEventToSession:612(9895ns) ForwardSessionEventToSession:612(3238ns) ForwardSessionEventToSession:612(3265ns) ForwardSessionEventToSession:612(5974ns) ForwardSessionEventToSession:612(3528ns) ForwardSessionEventToSession:612(8274ns) ForwardSessionEventToSession:612(2208ns)}}}
2025-11-11T17:21:12.087153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519127857984807:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.130810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132173935138:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.161516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519126101421963:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.118657Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519127857984807:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:12.189840Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519128710986661:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.174061Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132173935137:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.181952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519127878967761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.198878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132173935138:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.183058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519126017863240:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.190226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519129992987669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-11T17:21:12.199380Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519127878967831:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.199436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519127878967831:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.199604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519127878967762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.203649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519124416019285:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.183116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519126017863240:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.139034Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519132152952111:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.207315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132173935142:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-11T17:21:12.204334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987655:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.183712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519130312830550:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.217892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519130396389339:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
2025-11-11T17:21:12.158042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519127857984734:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.204366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519129992987655:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.213902Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519127878967831:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.208667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519127857984802:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.208724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:12.208776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519127857984802:2111] [node 6] ICP22 created new session: [5:7571519132152952115:2111]
2025-11-11T17:21:12.208817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519132152952115:2111] [node 6] ICS09 handshake done sender: [5:7571519127857984807:2113] self: [5:7571519127857984806:0] peer: [6:7571519128710986659:0] socket: 587 qp: -1
2025-11-11T17:21:12.208848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519132152952115:2111] [node 6] ICS10 traffic start
2025-11-11T17:21:12.208883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:12.210070Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519132152952110:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.226240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952114:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:12.227123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519127857984736:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.227224Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519132152952111:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:12.227590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519127857984734:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.227644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:12.227694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519127857984734:2048] [node 1] ICP22 created new session: [5:7571519132152952119:2048]
2025-11-11T17:21:12.227737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519132152952119:2048] [node 1] ICS09 handshake done sender: [5:7571519132152952111:2117] self: [5:7571519132152952113:0] peer: [1:7571519126017863235:0] socket: 605 qp: -1
2025-11-11T17:21:12.227770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519132152952119:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:12.227802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:12.250110Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519127878967833:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.250140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519127878967833:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.250675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519127878967827:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.250765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:12.250817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519127878967827:2107] [node 4] ICP22 created new session: [3:7571519132173935145:2107]
2025-11-11T17:21:12.250862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519132173935145:2107] [node 4] ICS09 handshake done sender: [3:7571519127878967831:2111] self: [3:7571519127878967830:0] peer: [4:7571519130396389338:0] socket: 596 qp: -1
2025-11-11T17:21:12.250895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519132173935145:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:12.250931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:12.252076Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519132173935138:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:12.252597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132173935144:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-11T17:21:12.252696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519127878967762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.252738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:12.252779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519127878967762:2049] [node 2] ICP22 created new session: [3:7571519132173935148:2049]
2025-11-11T17:21:12.252819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519132173935148:2049] [node 2] ICS09 handshake done sender: [3:7571519132173935138:2117] self: [3:7571519132173935141:0] peer: [2:7571519129992987654:0] socket: 619 qp: -1
2025-11-11T17:21:12.252851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519132173935148:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:12.252881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:12.258015Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519127878967833:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:12.261971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519127878967828:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.262017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:12.262075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519127878967828:2108] [node 5] ICP22 created new session: [3:7571519132173935151:2108]
2025-11-11T17:21:12.262149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519132173935151:2108] [node 5] ICS09 handshake done sender: [3:7571519127878967833:2112] self: [3:7571519127878967832:0] peer: [5:7571519132152952118:0] socket: 597 qp: -1
2025-11-11T17:21:12.262193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519132173935151:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:12.262251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:12.272663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519127878967835:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.272682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519127878967835:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.260960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986667:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.205234Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987660:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.183818Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519126017863238:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.282956Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.183835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519126017863238:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.286755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132173935144:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.287307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519127878967828:2108] [node 5] ICP18 handshake (actor [3:7571519132173935144:2119]) is too old
2025-11-11T17:21:12.283761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519126101421963:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519130396389336:2116] held: no
2025-11-11T17:21:12.191751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519126017863236:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.191796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519126017863236:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.201084Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519126017863240:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:12.201477Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519126017863236:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:12.234262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519126017863168:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.234330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:12.241799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519126017863168:2048] [node 2] ICP22 created new session: [1:7571519130312830555:2048]
2025-11-11T17:21:12.241860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519130312830555:2048] [node 2] ICS09 handshake done sender: [1:7571519126017863240:2113] self: [1:7571519126017863239:0] peer: [2:7571519129992987666:0] socket: 604 qp: -1
2025-11-11T17:21:12.241914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519130312830555:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:12.242000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:12.242476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519126017863233:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.242530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:12.242574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519126017863233:2107] [node 5] ICP22 created new session: [1:7571519130312830557:2107]
2025-11-11T17:21:12.242616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519130312830557:2107] [node 5] ICS09 handshake done sender: [1:7571519126017863236:2111] self: [1:7571519126017863235:0] peer: [5:7571519132152952113:0] socket: 602 qp: -1
2025-11-11T17:21:12.242671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519130312830557:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:12.242702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:12.282422Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519126017863238:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.280035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:12.288002Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519127878967835:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:12.205258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519129992987660:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.272395Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519128710986667:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.284016Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519130396389336:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:12.347058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519126017863169:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.347174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:12.347235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519126017863169:2049] [node 3] ICP22 created new session: [1:7571519130312830562:2049]
2025-11-11T17:21:12.347285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519130312830562:2049] [node 3] ICS09 handshake done sender: [1:7571519126017863238:2112] self: [1:7571519126017863237:0] peer: [3:7571519132173935140:0] socket: 603 qp: -1
2025-11-11T17:21:12.347326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519130312830562:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:12.347362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.290577Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519132152952110:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:12.272482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519124416019286:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.314254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132173935154:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-11T17:21:12.330587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519127878967829:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.330655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:12.330711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519127878967829:2109] [node 6] ICP22 created new session: [3:7571519132173935155:2109]
2025-11-11T17:21:12.330761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519132173935155:2109] [node 6] ICS09 handshake done sender: [3:7571519127878967835:2113] self: [3:7571519127878967834:0] peer: [6:7571519128710986668:0] socket: 632 qp: -1
2025-11-11T17:21:12.330804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519132173935155:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:12.330838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:12.331207Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519132173935137:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:12.349855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519127878967761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.349904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:12.349949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519127878967761:2048] [node 1] ICP22 created new session: [3:7571519132173935157:2048]
2025-11-11T17:21:12.349975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519132173935157:2048] [node 1] ICS09 handshake done sender: [3:7571519132173935137:2116] self: [3:7571519132173935140:0] peer: [1:7571519126017863237:0] socket: 607 qp: -1
2025-11-11T17:21:12.350001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519132173935157:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.310152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519126101421963:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.350028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.206208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519129992987671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-11T17:21:12.305269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519127857984736:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.322030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986669:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.221709Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519129992987655:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:12.245819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519125698020355:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(237098369ns) {Configure:104(237091316ns) {ProcessConfigured:135(16755509ns) {StartInitialHandshake:157(16753284ns) {DropHandshakes:469(2192ns) {DropIncomingHandshake:441(514ns) DropOutgoingHandshake:456(637ns)} GenerateSessionVirtualId:988(979ns) PrepareNewSessionHandshake:476(4536ns) {SwitchToState:178(1939ns)}}}}}
2025-11-11T17:21:12.265899Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519129992987660:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:12.270565Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987669:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.270687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519125698020291:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.270739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:12.270786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519125698020291:2049] [node 3] ICP22 created new session: [2:7571519129992987675:2049]
2025-11-11T17:21:12.270833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519129992987675:2049] [node 3] ICS09 handshake done sender: [2:7571519129992987655:2111] self: [2:7571519129992987654:0] peer: [3:7571519132173935141:0] socket: 615 qp: -1
2025-11-11T17:21:12.270863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519129992987675:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:12.270892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:12.271517Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21909) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.273905Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519129992987658:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:12.301717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519125698020290:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21909) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519129992987658:2114] held: no
2025-11-11T17:21:12.301837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519125698020290:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.301889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:12.301945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519125698020290:2048] [node 1] ICP22 created new session: [2:7571519129992987677:2048]
2025-11-11T17:21:12.302005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519129992987677:2048] [node 1] ICS09 handshake done sender: [2:7571519129992987658:2114] self: [2:7571519129992987666:0] peer: [1:7571519126017863239:0] socket: 611 qp: -1
2025-11-11T17:21:12.302049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519129992987677:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:12.302094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:12.302513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519125698020357:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.302551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:12.302594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519125698020357:2110] [node 6] ICP22 created new session: [2:7571519129992987679:2110]
2025-11-11T17:21:12.302643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519129992987679:2110] [node 6] ICS09 handshake done sender: [2:7571519129992987660:2115] self: [2:7571519129992987659:0] peer: [6:7571519128710986665:0] socket: 616 qp: -1
2025-11-11T17:21:12.302677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519129992987679:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:12.302708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:12.303453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519125698020357:2110] [node 6] ICP18 handshake (actor [2:7571519129992987669:2117]) is too old
2025-11-11T17:21:12.367362Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987662:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.367399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519129992987662:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.322138Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.325927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519124416019285:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519128710986661:2117] held: no
2025-11-11T17:21:12.326219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986670:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.328770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986671:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.328870Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519128710986667:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:12.354602Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519128710986669:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.354656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519124416019286:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.354713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:12.354761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519124416019286:2050] [node 3] ICP22 created new session: [6:7571519128710986673:2050]
2025-11-11T17:21:12.354806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519128710986673:2050] [node 3] ICS09 handshake done sender: [6:7571519128710986667:2119] self: [6:7571519128710986668:0] peer: [3:7571519127878967834:0] socket: 633 qp: -1
2025-11-11T17:21:12.354848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519128710986673:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:12.354885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:12.355535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986666:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.366919Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519128710986671:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.367004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519124416019284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.367229Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519128710986661:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:12.365566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132173935159:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-11T17:21:12.310208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:12.310260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519126101421963:2050] [node 3] ICP22 created new session: [4:7571519130396389342:2050]
2025-11-11T17:21:12.310309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519130396389342:2050] [node 3] ICS09 handshake done sender: [4:7571519130396389336:2116] self: [4:7571519130396389338:0] peer: [3:7571519127878967830:0] socket: 608 qp: -1
2025-11-11T17:21:12.310348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519130396389342:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:12.310384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:12.310930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519130396389340:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
2025-11-11T17:21:12.311398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519130396389341:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
2025-11-11T17:21:12.363802Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519130396389341:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.363936Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519130396389340:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.364261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519130396389345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
2025-11-11T17:21:12.364420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519126101421961:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.381735Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519130396389345:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.385347Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:12.305315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:12.397802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519124416019285:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.397869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:12.397945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519124416019285:2049] [node 2] ICP22 created new session: [6:7571519128710986676:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.385766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519130312830546:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.414073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519129992987670:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.414119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519129992987670:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.305359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519127857984736:2050] [node 3] ICP22 created new session: [5:7571519132152952124:2050]
2025-11-11T17:21:12.418873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519129992987670:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.398001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519128710986676:2049] [node 2] ICS09 handshake done sender: [6:7571519128710986661:2117] self: [6:7571519128710986665:0] peer: [2:7571519129992987659:0] socket: 617 qp: -1
2025-11-11T17:21:12.398033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519128710986676:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:12.398067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:12.422543Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21909) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.429876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519124416019350:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.439420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519124416019284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21909) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519128710986669:2120] held: no
2025-11-11T17:21:12.401928Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21909) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.305396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519132152952124:2050] [node 3] ICS09 handshake done sender: [5:7571519132152952110:2116] self: [5:7571519132152952118:0] peer: [3:7571519127878967832:0] socket: 598 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.385807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519130312830546:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.445821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519124416019350:2096] [node 4] ICP35 event processing took too much time PendingConnection:278(39948159ns) {IncomingHandshake:261(26656569ns) {DropIncomingHandshake:441(476ns) PrepareNewSessionHandshake:476(6729635ns) {SwitchToState:178(4451ns)} GenerateSessionVirtualId:988(1504ns) IssueIncomingHandshakeReply:193(8129841ns) {SwitchToState:178(4998ns)}}}
2025-11-11T17:21:12.425151Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519129992987662:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:12.403670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519126101421962:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.393697Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519126017863242:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.305429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519132152952124:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:12.442102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519125698020356:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.451860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986679:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.305466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:12.318053Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.403793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519126101421961:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21909) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519130396389341:2119] held: no
2025-11-11T17:21:12.404318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519130396389348:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
2025-11-11T17:21:12.424293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519130396389350:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8300
2025-11-11T17:21:12.424962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519130396389341:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:12.438038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519126101421961:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.438098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:12.438154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519126101421961:2048] [node 1] ICP22 created new session: [4:7571519130396389352:2048]
2025-11-11T17:21:12.438203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519130396389352:2048] [node 1] ICS09 handshake done sender: [4:7571519130396389341:2119] self: [4:7571519130396389346:0] peer: [1:7571519126017863241:0] socket: 582 qp: -1
2025-11-11T17:21:12.438239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519130396389352:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:12.438279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:12.438732Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519130396389345:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:12.467944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519126101421962:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.468024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:12.442168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:12.442228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519125698020356:2109] [node 5] ICP22 created new session: [2:7571519129992987685:2109]
2025-11-11T17:21:12.442275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519129992987685:2109] [node 5] ICS09 handshake done sender: [2:7571519129992987662:2116] self: [2:7571519129992987661:0] peer: [5:7571519132152952126:0] socket: 593 qp: -1
2025-11-11T17:21:12.442314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519129992987685:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:12.442354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:12.445963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519125698020355:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.446029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:12.446075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519125698020355:2108] [node 4] ICP22 created new session: [2:7571519129992987687:2108]
2025-11-11T17:21:12.446120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519129992987687:2108] [node 4] ICS09 handshake done sender: [2:7571519129992987670:2118] self: [2:7571519129992987668:0] peer: [4:7571519130396389349:0] socket: 641 qp: -1
2025-11-11T17:21:12.446156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519129992987687:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:12.446198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:12.468077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519126101421962:2049] [node 2] ICP22 created new session: [4:7571519130396389354:2049]
2025-11-11T17:21:12.468131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519130396389354:2049] [node 2] ICS09 handshake done sender: [4:7571519130396389345:2120] self: [4:7571519130396389349:0] peer: [2:7571519129992987668:0] socket: 642 qp: -1
2025-11-11T17:21:12.468176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519130396389354:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:12.468221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:12.468999Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.477664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519126101422033:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.477725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519126101422033:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.478503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519126101421962:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:12.393732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519126017863242:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.395020Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519130312830546:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:12.395367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519126017863242:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:12.348999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952123:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:12.349464Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519132152952123:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.349576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519127857984735:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.409913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519127857984736:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(24823291ns) {HandleHandshakeStatus:329(24821375ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(3989ns) SwitchToState:178(397ns) ProcessPendingSessionEvents:495(24496284ns) {ForwardSessionEventToSession:612(345333ns) ForwardSessionEventToSession:612(4008ns) ForwardSessionEventToSession:612(10066825ns) ForwardSessionEventToSession:612(4621ns) ForwardSessionEventToSession:612(2689931ns) ForwardSessionEventToSession:612(6540ns) ForwardSessionEventToSession:612(7135528ns) ForwardSessionEventToSession:612(4211961ns) ForwardSessionEventToSession:612(4810ns) ForwardSessionEventToSession:612(3185ns)}}}
2025-11-11T17:21:12.422351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952127:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:12.450321Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:12.457746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519127857984736:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.484435Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519126101422033:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:12.482693Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519128710986669:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:12.483077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519124416019284:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.483130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:12.483175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519124416019284:2048] [node 1] ICP22 created new session: [6:7571519128710986681:2048]
2025-11-11T17:21:12.483242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519128710986681:2048] [node 1] ICS09 handshake done sender: [6:7571519128710986669:2120] self: [6:7571519128710986675:0] peer: [1:7571519130312830545:0] socket: 637 qp: -1
2025-11-11T17:21:12.483281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519128710986681:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:12.483313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:12.483846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519128710986680:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49908
2025-11-11T17:21:12.485153Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519128710986671:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:12.485392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519124416019350:2096] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.485444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:12.485492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519124416019350:2096] [node 4] ICP22 created new session: [6:7571519128710986683:2096]
2025-11-11T17:21:12.485545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519128710986683:2096] [node 4] ICS09 handshake done sender: [6:7571519128710986671:2122] self: [6:7571519128710986678:0] peer: [4:7571519126101422032:0] socket: 620 qp: -1
2025-11-11T17:21:12.485583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519128710986683:2096] [node 4] ICS10 traffic start
2025-11-11T17:21:12.485637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:12.395609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519126017863232:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.395668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:12.395716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519126017863232:2109] [node 6] ICP22 created new session: [1:7571519130312830566:2109]
2025-11-11T17:21:12.395759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519130312830566:2109] [node 6] ICS09 handshake done sender: [1:7571519130312830546:2118] self: [1:7571519130312830545:0] peer: [6:7571519128710986675:0] socket: 636 qp: -1
2025-11-11T17:21:12.395793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519130312830566:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:12.395832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:12.396199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519126017863231:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.396272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:12.396315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519126017863231:2108] [node 4] ICP22 created new session: [1:7571519130312830568:2108]
2025-11-11T17:21:12.396360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519130312830568:2108] [node 4] ICS09 handshake done sender: [1:7571519126017863242:2114] self: [1:7571519126017863241:0] peer: [4:7571519130396389346:0] socket: 585 qp: -1
2025-11-11T17:21:12.396387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519130312830568:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:12.396416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:12.498161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519126101422027:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.498246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:12.498298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519126101422027:2109] [node 6] ICP22 created new session: [4:7571519130396389357:2109]
2025-11-11T17:21:12.498345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519130396389357:2109] [node 6] ICS09 handshake done sender: [4:7571519126101422033:2113] self: [4:7571519126101422032:0] peer: [6:7571519128710986678:0] socket: 640 qp: -1
2025-11-11T17:21:12.498382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519130396389357:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:12.498425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:12.486243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519127857984735:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519132152952123:2120] held: no
2025-11-11T17:21:12.491166Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519132152952123:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.502095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952128:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:12.506317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519124416019350:2096] [node 4] ICP35 event processing took too much time PendingConnection:278(20643511ns) {HandleHandshakeStatus:329(20641326ns) {DropIncomingHandshake:441(1108ns) DropOutgoingHandshake:456(5378ns) SwitchToState:178(799ns) ProcessPendingSessionEvents:495(12160286ns) {ForwardSessionEventToSession:612(12098674ns) ForwardSessionEventToSession:612(12890ns) ForwardSessionEventToSession:612(3875ns) ForwardSessionEventToSession:612(5894ns) ForwardSessionEventToSession:612(5959ns) ForwardSessionEventToSession:612(5587ns) ForwardSessionEventToSession:612(3943ns) ForwardSessionEventToSession:612(2714ns) ForwardSessionEventToSession:612(5338ns) ForwardSessionEventToSession:612(3094ns)}}}
2025-11-11T17:21:12.510765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519127857984735:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(38102602ns) {HandleHandshakeStatus:395(38098154ns) {LogHandshakeFail:472(16654ns) {UpdateErrorStateLog:544(5345ns)} DropOutgoingHandshake:456(2616ns) ProcessPendingSessionEvents:495(14094599ns) {DropSessionEvent:512(13436ns) DropSessionEvent:512(6021ns) DropSessionEvent:512(4953ns) DropSessionEvent:512(3252ns) DropSessionEvent:512(7652666ns) DropSessionEvent:512(3659ns) DropSessionEvent:512(7419ns) DropSessionEvent:512(29131ns) DropSessionEvent:512(6074ns) DropSessionEvent:512(6029ns)}}}
2025-11-11T17:21:12.510829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519127857984735:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.510877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:12.510928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519127857984735:2049] [node 2] ICP22 created new session: [5:7571519132152952129:2049]
2025-11-11T17:21:12.510986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519132152952129:2049] [node 2] ICS09 handshake done sender: [5:7571519132152952123:2120] self: [5:7571519132152952126:0] peer: [2:7571519129992987661:0] socket: 639 qp: -1
2025-11-11T17:21:12.511021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519132152952129:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:12.511055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:12.512363Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519126101422039:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.512404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519126101422039:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:12.513600Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519126101422039:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:12.511828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519132152952128:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:12.512004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519127857984801:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:12.514301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519132152952132:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8314
2025-11-11T17:21:12.517687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519126101422024:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:12.517783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:12.517833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519126101422024:2108] [node 5] ICP22 created new session: [4:7571519130396389360:2108]
2025-11-11T17:21:12.517917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519130396389360:2108] [node 5] ICS09 handshake done sender: [4:7571519126101422039:2115] self: [4:7571519126101422038:0] peer: [5:7571519132152952131:0] socket: 645 qp: -1
2025-11-11T17:21:12.517965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519130396389360:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:12.518002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:12.518823Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519132152952128:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:12.519135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519127857984801:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:12.519177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:12.519217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519127857984801:2110] [node 4] ICP22 created new session: [5:7571519132152952133:2110]
2025-11-11T17:21:12.519259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519132152952133:2110] [node 4] ICS09 handshake done sender: [5:7571519132152952128:2122] self: [5:7571519132152952131:0] peer: [4:7571519126101422038:0] socket: 646 qp: -1
2025-11-11T17:21:12.519294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519132152952133:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:12.519323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:12.565759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:12.566019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519129992987677:2048] [node 1] ICS01 socket: 611 reason# EndOfStream
2025-11-11T17:21:12.566063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519125698020290:2048] [node 1] ICP30 unregister session Session# [2:7571519129992987677:2048] VirtualId# [2:7571519129992987666:0]
2025-11-11T17:21:12.566100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519129992987677:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.566225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:12.566277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:12.569746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:12.569991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519132173935157:2048] [node 1] ICS01 socket: 607 reason# EndOfStream
2025-11-11T17:21:12.570035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519127878967761:2048] [node 1] ICP30 unregister session Session# [3:7571519132173935157:2048] VirtualId# [3:7571519132173935140:0]
2025-11-11T17:21:12.570071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519132173935157:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.570172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:12.570216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:12.573769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:12.573996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519132152952119:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-11-11T17:21:12.574048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519127857984734:2048] [node 1] ICP30 unregister session Session# [5:7571519132152952119:2048] VirtualId# [5:7571519132152952113:0]
2025-11-11T17:21:12.574099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519132152952119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.574185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:12.574227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:12.593720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:12.585738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:12.586005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519130396389352:2048] [node 1] ICS01 socket: 582 reason# EndOfStream
2025-11-11T17:21:12.586052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519126101421961:2048] [node 1] ICP30 unregister session Session# [4:7571519130396389352:2048] VirtualId# [4:7571519130396389346:0]
2025-11-11T17:21:12.586089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519130396389352:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.586199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:12.586247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:12.613915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519128710986681:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-11-11T17:21:12.613983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519124416019284:2048] [node 1] ICP30 unregister session Session# [6:7571519128710986681:2048] VirtualId# [6:7571519128710986675:0]
2025-11-11T17:21:12.614033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519128710986681:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.614152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:12.614204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:12.677109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:12.677348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519132173935148:2049] [node 2] ICS01 socket: 619 reason# EndOfStream
2025-11-11T17:21:12.677391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519127878967762:2049] [node 2] ICP30 unregister session Session# [3:7571519132173935148:2049] VirtualId# [3:7571519132173935141:0]
2025-11-11T17:21:12.677428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519132173935148:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.677525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:12.677570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:12.689791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:12.689790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:12.690053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519128710986676:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-11-11T17:21:12.690107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519124416019285:2049] [node 2] ICP30 unregister session Session# [6:7571519128710986676:2049] VirtualId# [6:7571519128710986665:0]
2025-11-11T17:21:12.690145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519128710986676:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.690239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:12.690294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:12.690019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519132152952129:2049] [node 2] ICS01 socket: 639 reason# EndOfStream
2025-11-11T17:21:12.690065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519127857984735:2049] [node 2] ICP30 unregister session Session# [5:7571519132152952129:2049] VirtualId# [5:7571519132152952126:0]
2025-11-11T17:21:12.690103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519132152952129:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.690186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:12.690235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:12.693779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:12.694108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519130396389354:2049] [node 2] ICS01 socket: 642 reason# EndOfStream
2025-11-11T17:21:12.694171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519126101421962:2049] [node 2] ICP30 unregister session Session# [4:7571519130396389354:2049] VirtualId# [4:7571519130396389349:0]
2025-11-11T17:21:12.694226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519130396389354:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.694349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:12.694427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:12.746004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:12.746221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519130396389342:2050] [node 3] ICS01 socket: 608 reason# EndOfStream
2025-11-11T17:21:12.746285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519126101421963:2050] [node 3] ICP30 unregister session Session# [4:7571519130396389342:2050] VirtualId# [4:7571519130396389338:0]
2025-11-11T17:21:12.746321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519130396389342:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.746404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:12.746443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:12.745445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:12.745752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519132152952124:2050] [node 3] ICS07 socket disconnect 598 reason# EndOfStream
2025-11-11T17:21:12.745857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519132152952124:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.745969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:12.746954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519132152952124:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:12.755333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:12.755563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519128710986673:2050] [node 3] ICS01 socket: 633 reason# EndOfStream
2025-11-11T17:21:12.755607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519124416019286:2050] [node 3] ICP30 unregister session Session# [6:7571519128710986673:2050] VirtualId# [6:7571519128710986668:0]
2025-11-11T17:21:12.755643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519128710986673:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.755751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:12.755795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:12.756506Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Couldn't connect to any resolved address
2025-11-11T17:21:12.756765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519127857984736:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:12.756817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519132152952124:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:12.756848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519127857984736:2050] [node 3] ICP30 unregister session Session# [5:7571519132152952124:2050] VirtualId# [5:7571519132152952118:0]
2025-11-11T17:21:12.756891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:12.756923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519127857984736:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:12.756938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:12.769745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519127857984736:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:12.833802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:12.834032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519132152952133:2110] [node 4] ICS01 socket: 646 reason# EndOfStream
2025-11-11T17:21:12.834079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519127857984801:2110] [node 4] ICP30 unregister session Session# [5:7571519132152952133:2110] VirtualId# [5:7571519132152952131:0]
2025-11-11T17:21:12.834111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519132152952133:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.834202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:12.834250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:12.834394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:12.834550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519128710986683:2096] [node 4] ICS07 socket disconnect 620 reason# EndOfStream
2025-11-11T17:21:12.834575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519128710986683:2096] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.834624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:12.834662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519128710986683:2096] [node 4] ICS15 start handshake
2025-11-11T17:21:12.835903Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27680) Couldn't connect to any resolved address
2025-11-11T17:21:12.836117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519124416019350:2096] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27680) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:12.836160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519128710986683:2096] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:12.836197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519124416019350:2096] [node 4] ICP30 unregister session Session# [6:7571519128710986683:2096] VirtualId# [6:7571519128710986678:0]
2025-11-11T17:21:12.836233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:12.836260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519124416019350:2096] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27680) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:12.836278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27680) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:12.845296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519124416019350:2096] [node 4] ICP33 wake up from error state
2025-11-11T17:21:12.891645Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:12.891903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519128710986662:2091] [node 5] ICS01 socket: 588 reason# EndOfStream
2025-11-11T17:21:12.891947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519124416019329:2091] [node 5] ICP30 unregister session Session# [6:7571519128710986662:2091] VirtualId# [6:7571519128710986659:0]
2025-11-11T17:21:12.891987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519128710986662:2091] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:12.892097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:12.892141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:13.015610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519135244883683:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:13.015649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519135244883684:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:13.054032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519133040549284:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:13.054062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519133040549283:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:13.078606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519132771241192:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:13.078739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519132771241191:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:13.101079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519135794101184:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:13.101112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519135794101185:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:13.101136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519135794101186:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:13.152064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519133688570518:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:13.152102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519133688570519:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:13.152113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519133688570520:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:13.175295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519133983455848:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:13.175356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519133983455849:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:13.175482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519133983455850:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:13.407916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519133688570544:2068] [node 6] ICP01 ready to work
2025-11-11T17:21:13.378395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519135794101203:2064] [node 5] ICP01 ready to work
2025-11-11T17:21:13.516676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519133688570519:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(108433907ns) {Configure:104(108427523ns) {ProcessConfigured:135(92935ns) {StartInitialHandshake:157(88670ns) {DropHandshakes:469(4622ns) {DropIncomingHandshake:441(1148ns) DropOutgoingHandshake:456(622ns)} GenerateSessionVirtualId:988(1244ns) PrepareNewSessionHandshake:476(2456ns) {SwitchToState:178(859ns)}}}}}
2025-11-11T17:21:13.517111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519133688570548:2073] [node 4] ICP01 ready to work
2025-11-11T17:21:13.643874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519133983455848:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(310422634ns) {Configure:104(310416560ns) {ProcessConfigured:135(93766ns) {StartInitialHandshake:157(90658ns) {DropHandshakes:469(2945ns) {DropIncomingHandshake:441(517ns) DropOutgoingHandshake:456(598ns)} GenerateSessionVirtualId:988(813ns) PrepareNewSessionHandshake:476(1731ns) {SwitchToState:178(573ns)}}}}}
2025-11-11T17:21:13.621322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519133040549283:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(269123110ns) {Configure:104(269117578ns) {ProcessConfigured:135(78570ns) {StartInitialHandshake:157(75188ns) {DropHandshakes:469(3138ns) {DropIncomingHandshake:441(898ns) DropOutgoingHandshake:456(507ns)} GenerateSessionVirtualId:988(1180ns) PrepareNewSessionHandshake:476(1500ns) {SwitchToState:178(680ns)}}}}}
2025-11-11T17:21:13.661536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519135244883744:2104] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.617026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519135794101203:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(213225936ns) {Configure:104(213215786ns) {ProcessConfigured:135(87760ns) {StartInitialHandshake:157(84065ns) {DropHandshakes:469(4228ns) {DropIncomingHandshake:441(802ns) DropOutgoingHandshake:456(745ns)} GenerateSessionVirtualId:988(1095ns) PrepareNewSessionHandshake:476(2497ns) {SwitchToState:178(779ns)}}}}}
2025-11-11T17:21:13.629739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519133040549322:2081] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.693567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519133688570556:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:13.678369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519135794101210:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:13.665639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519132771241207:2060] [node 4] ICP01 ready to work
2025-11-11T17:21:13.753804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519135244883750:2103] [node 4] ICP01 ready to work
2025-11-11T17:21:13.777910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519135244883683:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(342308794ns) {Configure:104(342303102ns) {ProcessConfigured:135(13407335ns) {StartInitialHandshake:157(13403902ns) {DropHandshakes:469(2822ns) {DropIncomingHandshake:441(598ns) DropOutgoingHandshake:456(560ns)} GenerateSessionVirtualId:988(755ns) PrepareNewSessionHandshake:476(5769ns) {SwitchToState:178(3022ns)}}}}}
2025-11-11T17:21:13.632011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519133040549284:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(272919561ns) {Configure:104(272913674ns) {ProcessConfigured:135(66319ns) {StartInitialHandshake:157(62605ns) {DropHandshakes:469(4568ns) {DropIncomingHandshake:441(936ns) DropOutgoingHandshake:456(826ns)} GenerateSessionVirtualId:988(1045ns) PrepareNewSessionHandshake:476(1965ns) {SwitchToState:178(663ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.673799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519132771241212:2061] [node 6] ICP01 ready to work
2025-11-11T17:21:13.794128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519135244883753:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.632785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519133040549329:2088] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.773895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519133983455917:2088] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.679289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519135794101186:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(237287005ns) {Configure:104(237280706ns) {ProcessConfigured:135(78373ns) {StartInitialHandshake:157(75271ns) {DropHandshakes:469(3954ns) {DropIncomingHandshake:441(903ns) DropOutgoingHandshake:456(699ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:476(1854ns) {SwitchToState:178(799ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.791242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519133983455901:2090] [node 4] ICP01 ready to work
2025-11-11T17:21:13.685714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519132771241213:2062] [node 5] ICP01 ready to work
2025-11-11T17:21:13.842842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519133688570556:2084] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.752487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519133040549341:2080] [node 4] ICP01 ready to work
2025-11-11T17:21:13.741834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519135794101242:2066] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.930752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519135244883759:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59231
2025-11-11T17:21:13.858793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132771241266:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41232
2025-11-11T17:21:13.841952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519133040549328:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.823094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519135794101210:2070] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.883691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519133688570548:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:13.831743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519133983455921:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.930915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519135244883763:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59231
2025-11-11T17:21:13.883788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519133688570593:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:13.931257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519133040549329:2088] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(297988976ns) {Configure:104(297983996ns) {ProcessConfigured:135(28963530ns) {StartInitialHandshake:157(28959689ns) {DropHandshakes:469(12811873ns) {DropIncomingHandshake:441(803ns) DropOutgoingHandshake:456(2364ns)} GenerateSessionVirtualId:988(3359ns) PrepareNewSessionHandshake:476(5007ns) {SwitchToState:178(2187ns)}}}}}
2025-11-11T17:21:13.848195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519135794101258:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:13.944225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519135244883750:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(133232560ns) {Configure:104(133227193ns) {ProcessConfigured:135(39776503ns) {StartInitialHandshake:157(39769831ns) {DropHandshakes:469(4175ns) {DropIncomingHandshake:441(909ns) DropOutgoingHandshake:456(949ns)} GenerateSessionVirtualId:988(2963ns) PrepareNewSessionHandshake:476(7947830ns) {SwitchToState:178(8249ns)}}}}}
2025-11-11T17:21:13.965737Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883763:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.859010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132771241267:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41232
2025-11-11T17:21:13.875445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519132771241191:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(105683248ns) {Configure:104(105677204ns) {ProcessConfigured:135(66651ns) {StartInitialHandshake:157(64494ns) {DropHandshakes:469(1558ns) {DropIncomingHandshake:441(153ns) DropOutgoingHandshake:456(167ns)} GenerateSessionVirtualId:988(350ns) PrepareNewSessionHandshake:476(2239ns) {SwitchToState:178(911ns)}}}}}
2025-11-11T17:21:13.897894Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132771241266:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.925871Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132771241267:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.926025Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132771241260:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.926047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519132771241260:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:13.961766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132771241258:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.961838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519132771241258:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:13.968367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519132771241192:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(226410947ns) {Configure:104(214512946ns) {ProcessConfigured:135(76884ns) {StartInitialHandshake:157(72931ns) {DropHandshakes:469(3802ns) {DropIncomingHandshake:441(863ns) DropOutgoingHandshake:456(689ns)} GenerateSessionVirtualId:988(1185ns) PrepareNewSessionHandshake:476(1844ns) {SwitchToState:178(684ns)}}}}}
2025-11-11T17:21:13.968948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519132771241273:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41232
2025-11-11T17:21:13.984128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519132771241260:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:13.831797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519133983455922:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:14.005789Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883759:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.883896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519133688570594:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:13.885657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519133983455882:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:13.910227Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519133983455882:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.910381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519133983455917:2088] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:13.917873Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519133983455922:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.918002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519133983455850:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:13.938730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519133983455929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:13.962006Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519133983455929:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.984334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519133983455849:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(220796370ns) {Configure:104(220790849ns) {ProcessConfigured:135(84057ns) {StartInitialHandshake:157(81861ns) {DropHandshakes:469(6106ns) {DropIncomingHandshake:441(1107ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(1046ns) PrepareNewSessionHandshake:476(2572ns) {SwitchToState:178(754ns)}}}}}
2025-11-11T17:21:14.003147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519133983455901:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.005680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519133983455921:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.931598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519133040549328:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.895883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519135794101184:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(66001044ns) {Configure:104(65989594ns) {ProcessConfigured:135(72792ns) {StartInitialHandshake:157(70462ns) {DropHandshakes:469(2652ns) {DropIncomingHandshake:441(881ns) DropOutgoingHandshake:456(397ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:476(2381ns) {SwitchToState:178(802ns)}}}}}
2025-11-11T17:21:13.896359Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519135794101208:2069] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.896386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519135794101208:2069] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:13.898836Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519135794101208:2069] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:13.899161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519135794101203:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:13.899240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:13.899288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519135794101203:2064] [node 5] ICP22 created new session: [4:7571519135794101266:2064]
2025-11-11T17:21:13.899334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519135794101266:2064] [node 5] ICS09 handshake done sender: [4:7571519135794101208:2069] self: [4:7571519135794101207:0] peer: [5:7571519133688570596:0] socket: 592 qp: -1
2025-11-11T17:21:13.899367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519135794101266:2064] [node 5] ICS10 traffic start
2025-11-11T17:21:13.899399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:13.906096Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519135794101258:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.917865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519135794101186:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.021973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519135794101185:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(342388141ns) {Configure:104(342383246ns) {ProcessConfigured:135(28037809ns) {StartInitialHandshake:157(28034841ns) {DropHandshakes:469(3093ns) {DropIncomingHandshake:441(712ns) DropOutgoingHandshake:456(521ns)} GenerateSessionVirtualId:988(1272ns) PrepareNewSessionHandshake:476(6496ns) {SwitchToState:178(4000ns)}}}}}
2025-11-11T17:21:14.025745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519135244883684:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(273776355ns) {Configure:104(273770481ns) {ProcessConfigured:135(47827390ns) {StartInitialHandshake:157(47823840ns) {DropHandshakes:469(3166ns) {DropIncomingHandshake:441(810ns) DropOutgoingHandshake:456(523ns)} GenerateSessionVirtualId:988(1267ns) PrepareNewSessionHandshake:476(5787ns) {SwitchToState:178(2992ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.023059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519133983455848:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.023177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519138278423229:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:14.023236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519138278423230:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:14.032960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519133983455901:2090] [node 4] ICP35 event processing took too much time PendingConnection:278(18770257ns) {IncomingHandshake:261(18767134ns) {DropIncomingHandshake:441(606ns) PrepareNewSessionHandshake:476(2453ns) {SwitchToState:178(916ns)} GenerateSessionVirtualId:988(645ns) IssueIncomingHandshakeReply:193(72266ns) {SwitchToState:178(802ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.003705Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519132771241258:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:13.883981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519133688570595:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:14.049826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132771241262:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.049869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519132771241262:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.041251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519135794101260:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.030113Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883752:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.042196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519133040549322:2081] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(291823330ns) {Configure:104(248879585ns) {ProcessConfigured:135(69773ns) {StartInitialHandshake:157(67417ns) {DropHandshakes:469(3226ns) {DropIncomingHandshake:441(1268ns) DropOutgoingHandshake:456(349ns)} GenerateSessionVirtualId:988(1211ns) PrepareNewSessionHandshake:476(7693ns) {SwitchToState:178(385ns)}}}}}
2025-11-11T17:21:13.898161Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519133688570594:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.057298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519138278423231:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:14.041329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519135794101260:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.030151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519135244883752:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.095755Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:14.096808Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519133983455922:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.923773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519133688570597:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.126194Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519133983455882:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:14.067652Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519132771241262:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.103354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519133040549361:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.030956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519135244883765:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59231
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.073823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519132771241207:2060] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:13.957992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519133688570518:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(619948346ns) {Configure:104(619943640ns) {ProcessConfigured:135(15612223ns) {StartInitialHandshake:157(15609558ns) {DropHandshakes:469(3321ns) {DropIncomingHandshake:441(748ns) DropOutgoingHandshake:456(748ns)} GenerateSessionVirtualId:988(546ns) PrepareNewSessionHandshake:476(6572ns) {SwitchToState:178(3126ns)}}}}}
2025-11-11T17:21:14.126601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519133983455850:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519133983455922:2115] held: no
2025-11-11T17:21:14.047059Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519135794101260:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.039160Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519135244883752:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.122736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519137335516662:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45417
2025-11-11T17:21:14.126728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519133983455850:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:13.958146Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14307) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:14.093789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519135794101242:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.073905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:14.123356Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519137335516662:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.047296Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883755:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.970566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519133688570520:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(584355590ns) {Configure:104(584350437ns) {ProcessConfigured:135(20277134ns) {StartInitialHandshake:157(20274011ns) {DropHandshakes:469(3827ns) {DropIncomingHandshake:441(919ns) DropOutgoingHandshake:456(485ns)} GenerateSessionVirtualId:988(928ns) PrepareNewSessionHandshake:476(3565ns) {SwitchToState:178(1845ns)}}}}}
2025-11-11T17:21:13.970835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519133688570550:2075] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.970862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519133688570550:2075] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:13.971631Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519133688570593:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:13.971838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519133688570548:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14307) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519133688570556:2084] held: no
2025-11-11T17:21:13.972094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519133688570520:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.010007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519133688570518:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.045690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519133688570595:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.049678Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519133688570550:2075] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:14.057262Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519133688570556:2084] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:14.066861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519133688570548:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.066912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:14.066967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519133688570548:2073] [node 4] ICP22 created new session: [5:7571519137983537904:2073]
2025-11-11T17:21:14.067020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519137983537904:2073] [node 4] ICS09 handshake done sender: [5:7571519133688570556:2084] self: [5:7571519133688570596:0] peer: [4:7571519135794101207:0] socket: 595 qp: -1
2025-11-11T17:21:14.067059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519137983537904:2073] [node 4] ICS10 traffic start
2025-11-11T17:21:14.067097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:14.090537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519133688570544:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.090600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:14.090651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519133688570544:2068] [node 6] ICP22 created new session: [5:7571519137983537906:2068]
2025-11-11T17:21:14.090707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519137983537906:2068] [node 6] ICS09 handshake done sender: [5:7571519133688570550:2075] self: [5:7571519133688570549:0] peer: [6:7571519133983455927:0] socket: 589 qp: -1
2025-11-11T17:21:14.090743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519137983537906:2068] [node 6] ICS10 traffic start
2025-11-11T17:21:14.090782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:14.091104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571519133688570544:2068] [node 6] ICP18 handshake (actor [5:7571519133688570595:2116]) is too old
2025-11-11T17:21:14.091360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519137983537901:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:14.125992Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519133688570593:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:14.155889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519133688570518:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.155912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519137983537903:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:14.165712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:14.175769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519133688570518:2048] [node 1] ICP22 created new session: [5:7571519137983537909:2048]
2025-11-11T17:21:14.180758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519137983537909:2048] [node 1] ICS09 handshake done sender: [5:7571519133688570593:2114] self: [5:7571519137983537900:0] peer: [1:7571519135244883751:0] socket: 611 qp: -1
2025-11-11T17:21:14.183820Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519133688570594:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:14.184120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519137983537909:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:14.093824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:14.073964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519132771241207:2060] [node 4] ICP22 created new session: [3:7571519137066208574:2060]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.126776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.151653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519133040549327:2090] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.193898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:14.047335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519135244883755:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.126823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519133983455850:2050] [node 3] ICP22 created new session: [6:7571519138278423233:2050]
2025-11-11T17:21:14.126866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519138278423233:2050] [node 3] ICS09 handshake done sender: [6:7571519133983455922:2115] self: [6:7571519133983455928:0] peer: [3:7571519132771241257:0] socket: 605 qp: -1
2025-11-11T17:21:14.126904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519138278423233:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:14.126935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:14.141701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519138278423229:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.147549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519133983455917:2088] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.147642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:14.147700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519133983455917:2088] [node 5] ICP22 created new session: [6:7571519138278423235:2088]
2025-11-11T17:21:14.147758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519138278423235:2088] [node 5] ICS09 handshake done sender: [6:7571519133983455882:2111] self: [6:7571519133983455927:0] peer: [5:7571519133688570549:0] socket: 591 qp: -1
2025-11-11T17:21:14.147797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519138278423235:2088] [node 5] ICS10 traffic start
2025-11-11T17:21:14.147858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:14.149030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519133983455849:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.151158Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18382) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:14.163594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519133983455917:2088] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:14.163823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519138278423239:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:14.183414Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:14.191436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519133983455848:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519133983455921:2114] held: no
2025-11-11T17:21:14.191671Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519133983455929:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:14.231703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519133983455901:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.237883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519138278423241:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:14.074021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519137066208574:2060] [node 4] ICS09 handshake done sender: [3:7571519132771241260:2111] self: [3:7571519132771241259:0] peer: [4:7571519135794101269:0] socket: 601 qp: -1
2025-11-11T17:21:14.093883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519135794101242:2066] [node 6] ICP22 created new session: [4:7571519140089068570:2066]
2025-11-11T17:21:14.151693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519133040549327:2090] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.229495Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:14.072998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519135244883744:2104] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.257996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519138278423240:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12365
2025-11-11T17:21:14.093928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519140089068570:2066] [node 6] ICS09 handshake done sender: [4:7571519135794101260:2116] self: [4:7571519135794101259:0] peer: [6:7571519133983455932:0] socket: 612 qp: -1
2025-11-11T17:21:14.074062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519137066208574:2060] [node 4] ICS10 traffic start
2025-11-11T17:21:14.077771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.154038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519133040549358:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.265680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:14.085665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883765:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.244549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519133688570520:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.074097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:14.093964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519140089068570:2066] [node 6] ICS10 traffic start
2025-11-11T17:21:14.277680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519133983455901:2090] [node 4] ICP22 created new session: [6:7571519138278423242:2090]
2025-11-11T17:21:14.262094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519133688570518:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(105979953ns) {HandleHandshakeStatus:329(105977300ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(27280ns) SwitchToState:178(710ns) ProcessPendingSessionEvents:495(12026795ns) {ForwardSessionEventToSession:612(11971250ns) ForwardSessionEventToSession:612(5600ns) ForwardSessionEventToSession:612(6441ns) ForwardSessionEventToSession:612(3095ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(5392ns) ForwardSessionEventToSession:612(3986ns) ForwardSessionEventToSession:612(10151ns) ForwardSessionEventToSession:612(5508ns) ForwardSessionEventToSession:612(2880ns)}}}
2025-11-11T17:21:14.265819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:14.265883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519133688570520:2050] [node 3] ICP22 created new session: [5:7571519137983537911:2050]
2025-11-11T17:21:14.265940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519137983537911:2050] [node 3] ICS09 handshake done sender: [5:7571519133688570594:2115] self: [5:7571519133688570602:0] peer: [3:7571519132771241261:0] socket: 614 qp: -1
2025-11-11T17:21:14.265979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519137983537911:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:14.266022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:14.266476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519133688570520:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(60200321ns) {HandleHandshakeStatus:329(34605649ns) {DropIncomingHandshake:441(2775ns) DropOutgoingHandshake:456(6089ns) SwitchToState:178(844ns) ProcessPendingSessionEvents:495(101475ns) {ForwardSessionEventToSession:612(51516ns) ForwardSessionEventToSession:612(6775ns) ForwardSessionEventToSession:612(3221ns) ForwardSessionEventToSession:612(4982ns) ForwardSessionEventToSession:612(6207ns) ForwardSessionEventToSession:612(3100ns) ForwardSessionEventToSession:612(5511ns) ForwardSessionEventToSession:612(2940ns) ForwardSessionEventToSession:612(4360ns) ForwardSessionEventToSession:612(5436ns)}}}
2025-11-11T17:21:14.266590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519133688570520:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:14.290362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519137983537913:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:14.298037Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519137983537913:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.298170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519133688570519:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.103222Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519135244883755:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:14.118029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519135244883744:2104] [node 5] ICP22 created new session: [1:7571519139539851065:2104]
2025-11-11T17:21:14.127690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519139539851065:2104] [node 5] ICS09 handshake done sender: [1:7571519135244883752:2112] self: [1:7571519135244883751:0] peer: [5:7571519137983537900:0] socket: 598 qp: -1
2025-11-11T17:21:14.136270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519139539851065:2104] [node 5] ICS10 traffic start
2025-11-11T17:21:14.149754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519139539851070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59231
2025-11-11T17:21:14.149994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519135244883753:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.150080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:14.150129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519135244883753:2105] [node 6] ICP22 created new session: [1:7571519139539851071:2105]
2025-11-11T17:21:14.150177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519139539851071:2105] [node 6] ICS09 handshake done sender: [1:7571519135244883755:2113] self: [1:7571519135244883754:0] peer: [6:7571519138278423232:0] socket: 599 qp: -1
2025-11-11T17:21:14.150211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519139539851071:2105] [node 6] ICS10 traffic start
2025-11-11T17:21:14.150252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:14.151042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519139539851069:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59231
2025-11-11T17:21:14.153731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:14.163012Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519139539851069:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.183762Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:14.238251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883762:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.238281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519135244883762:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.240363Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883749:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.240388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519135244883749:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.246008Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519135244883762:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:14.247658Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519135244883749:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:14.250083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519135244883744:2104] [node 5] ICP35 event processing took too much time PendingConnection:278(168475935ns) {HandleHandshakeStatus:329(168472300ns) {DropIncomingHandshake:441(475ns) DropOutgoingHandshake:456(919ns) SwitchToState:178(1106ns) ProcessPendingSessionEvents:495(35261977ns) {ForwardSessionEventToSession:612(35200148ns) ForwardSessionEventToSession:612(7374ns) ForwardSessionEventToSession:612(4158ns) ForwardSessionEventToSession:612(7260ns) ForwardSessionEventToSession:612(3356ns) ForwardSessionEventToSession:612(5755ns) ForwardSessionEventToSession:612(3544ns) ForwardSessionEventToSession:612(4799ns) ForwardSessionEventToSession:612(4940ns) ForwardSessionEventToSession:612(3448ns)}}}
2025-11-11T17:21:14.269851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519135244883750:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.269938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:14.269985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519135244883750:2103] [node 4] ICP22 created new session: [1:7571519139539851078:2103]
2025-11-11T17:21:14.270042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519139539851078:2103] [node 4] ICS09 handshake done sender: [1:7571519135244883762:2115] self: [1:7571519135244883760:0] peer: [4:7571519140089068575:0] socket: 637 qp: -1
2025-11-11T17:21:14.270076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519139539851078:2103] [node 4] ICS10 traffic start
2025-11-11T17:21:14.270105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:14.270810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519135244883744:2104] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:14.299227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519135244883683:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.299315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:14.299351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519135244883683:2048] [node 2] ICP22 created new session: [1:7571519139539851080:2048]
2025-11-11T17:21:14.299394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519139539851080:2048] [node 2] ICS09 handshake done sender: [1:7571519135244883749:2111] self: [1:7571519135244883745:0] peer: [2:7571519137335516667:0] socket: 597 qp: -1
2025-11-11T17:21:14.299442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519139539851080:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:14.299469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:14.346099Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519135244883764:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.346131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519135244883764:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.277751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519138278423242:2090] [node 4] ICS09 handshake done sender: [6:7571519133983455929:2117] self: [6:7571519133983455932:0] peer: [4:7571519135794101259:0] socket: 616 qp: -1
2025-11-11T17:21:14.347553Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519135244883764:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:14.277799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519138278423242:2090] [node 4] ICS10 traffic start
2025-11-11T17:21:14.074656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519132771241212:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.277849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:14.081975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:14.278283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519133983455901:2090] [node 4] ICP35 event processing took too much time PendingConnection:278(52082305ns) {HandleHandshakeStatus:329(52078632ns) {DropIncomingHandshake:441(5658ns) DropOutgoingHandshake:456(16628ns) SwitchToState:178(384ns) ProcessPendingSessionEvents:495(114107ns) {ForwardSessionEventToSession:612(57800ns) ForwardSessionEventToSession:612(4928ns) ForwardSessionEventToSession:612(11492ns) ForwardSessionEventToSession:612(2717ns) ForwardSessionEventToSession:612(4904ns) ForwardSessionEventToSession:612(5543ns) ForwardSessionEventToSession:612(6581ns) ForwardSessionEventToSession:612(6268ns) ForwardSessionEventToSession:612(2904ns) ForwardSessionEventToSession:612(2863ns)}}}
2025-11-11T17:21:14.082038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519132771241212:2061] [node 6] ICP22 created new session: [3:7571519137066208576:2061]
2025-11-11T17:21:14.278416Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519138278423229:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:14.082092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519137066208576:2061] [node 6] ICS09 handshake done sender: [3:7571519132771241258:2110] self: [3:7571519132771241257:0] peer: [6:7571519133983455928:0] socket: 600 qp: -1
2025-11-11T17:21:14.082134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519137066208576:2061] [node 6] ICS10 traffic start
2025-11-11T17:21:14.298191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519133983455849:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.082183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:14.298243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:14.091997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519132771241273:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.298290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519133983455849:2049] [node 2] ICP22 created new session: [6:7571519138278423245:2049]
2025-11-11T17:21:14.092041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519132771241213:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.298343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519138278423245:2049] [node 2] ICS09 handshake done sender: [6:7571519138278423229:2119] self: [6:7571519138278423237:0] peer: [2:7571519133040549357:0] socket: 629 qp: -1
2025-11-11T17:21:14.092069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:14.298378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519138278423245:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:14.298411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:14.092106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519132771241213:2062] [node 5] ICP22 created new session: [3:7571519137066208580:2062]
2025-11-11T17:21:14.300701Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519133983455921:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:14.300966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519133983455848:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.301004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:14.301041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519133983455848:2048] [node 1] ICP22 created new session: [6:7571519138278423247:2048]
2025-11-11T17:21:14.301083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519138278423247:2048] [node 1] ICS09 handshake done sender: [6:7571519133983455921:2114] self: [6:7571519138278423232:0] peer: [1:7571519135244883754:0] socket: 604 qp: -1
2025-11-11T17:21:14.301118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519138278423247:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:14.301146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:14.092144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519137066208580:2062] [node 5] ICS09 handshake done sender: [3:7571519132771241262:2112] self: [3:7571519132771241261:0] peer: [5:7571519133688570602:0] socket: 602 qp: -1
2025-11-11T17:21:14.092170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519137066208580:2062] [node 5] ICS10 traffic start
2025-11-11T17:21:14.092198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:14.127520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519132771241192:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.145211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519132771241192:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(34648789ns) {IncomingHandshake:261(34647408ns) {DropIncomingHandshake:441(307ns) PrepareNewSessionHandshake:476(1438ns) {SwitchToState:178(443ns)} GenerateSessionVirtualId:988(490ns) IssueIncomingHandshakeReply:193(25542836ns) {SwitchToState:178(5463ns)}}}
2025-11-11T17:21:14.164378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519137066208583:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41232
2025-11-11T17:21:14.164431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519137066208584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41232
2025-11-11T17:21:14.154067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519133040549358:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.093998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:14.169227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519137066208585:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41232
2025-11-11T17:21:14.170459Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519132771241273:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:14.094447Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:14.110934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519135794101186:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519135794101258:2115] held: no
2025-11-11T17:21:14.111308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519135794101270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:14.134196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519140089068572:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:14.142942Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519135794101258:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:14.166163Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519140089068572:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.166225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519135794101186:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.166281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:14.173984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519135794101184:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.177857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519135794101186:2050] [node 3] ICP22 created new session: [4:7571519140089068574:2050]
2025-11-11T17:21:14.194344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519140089068574:2050] [node 3] ICS09 handshake done sender: [4:7571519135794101258:2115] self: [4:7571519135794101269:0] peer: [3:7571519132771241259:0] socket: 606 qp: -1
2025-11-11T17:21:14.214778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519140089068574:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:14.244029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:14.246316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519140089068576:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:14.246474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519140089068578:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:14.246526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519140089068579:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:14.248182Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519140089068572:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:14.248930Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519140089068578:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.248971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519135794101184:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.249043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:14.249091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519135794101184:2048] [node 1] ICP22 created new session: [4:7571519140089068580:2048]
2025-11-11T17:21:14.249137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519140089068580:2048] [node 1] ICS09 handshake done sender: [4:7571519140089068572:2120] self: [4:7571519140089068575:0] peer: [1:7571519135244883760:0] socket: 607 qp: -1
2025-11-11T17:21:14.249169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519140089068580:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:14.249203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:14.293708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519135794101186:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(107508489ns) {HandleHandshakeStatus:329(107505846ns) {DropIncomingHandshake:441(742ns) DropOutgoingHandshake:456(400ns) SwitchToState:178(809ns) ProcessPendingSessionEvents:495(1326ns)}}
2025-11-11T17:21:14.305785Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:14.317991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519135794101242:2066] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:14.332433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519135794101185:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.347949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519140089068583:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58167
2025-11-11T17:21:14.361638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519135794101185:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(41727684ns) {IncomingHandshake:261(22629400ns) {DropIncomingHandshake:441(335ns) PrepareNewSessionHandshake:476(1829ns) {SwitchToState:178(499ns)} GenerateSessionVirtualId:988(853ns) IssueIncomingHandshakeReply:193(22591239ns) {SwitchToState:178(2674ns)}}}
2025-11-11T17:21:14.362779Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519140089068578:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:14.362999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519135794101185:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.363059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:14.363104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519135794101185:2049] [node 2] ICP22 created new session: [4:7571519140089068584:2049]
2025-11-11T17:21:14.363162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519140089068584:2049] [node 2] ICS09 handshake done sender: [4:7571519140089068578:2122] self: [4:7571519140089068582:0] peer: [2:7571519133040549359:0] socket: 609 qp: -1
2025-11-11T17:21:14.363198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519140089068584:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:14.363231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:14.377322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519135244883684:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.385746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:14.393697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519135244883684:2049] [node 3] ICP22 created new session: [1:7571519139539851083:2049]
2025-11-11T17:21:14.155440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519137335516664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45417
2025-11-11T17:21:14.155599Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519133040549358:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:14.155839Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519133040549361:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.169801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519133040549283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.170376Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519133040549327:2090] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:14.177814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519133040549329:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.212639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519133040549284:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.212747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:14.212809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519133040549284:2049] [node 3] ICP22 created new session: [2:7571519137335516668:2049]
2025-11-11T17:21:14.212858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519137335516668:2049] [node 3] ICS09 handshake done sender: [2:7571519133040549327:2090] self: [2:7571519133040549326:0] peer: [3:7571519137066208582:0] socket: 622 qp: -1
2025-11-11T17:21:14.212900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519137335516668:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:14.212940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:14.245665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:14.247352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519137335516672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45417
2025-11-11T17:21:14.248635Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519133040549361:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:14.260088Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519137335516664:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.260138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519133040549283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.260269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:14.260333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519133040549283:2048] [node 1] ICP22 created new session: [2:7571519137335516673:2048]
2025-11-11T17:21:14.260384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519137335516673:2048] [node 1] ICS09 handshake done sender: [2:7571519133040549361:2116] self: [2:7571519137335516667:0] peer: [1:7571519135244883745:0] socket: 619 qp: -1
2025-11-11T17:21:14.260418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519137335516673:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:14.260455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:14.273328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519133040549329:2088] [node 6] ICP22 created new session: [2:7571519137335516671:2088]
2025-11-11T17:21:14.313754Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519137335516660:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.313784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519137335516660:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.318522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519137335516671:2088] [node 6] ICS09 handshake done sender: [2:7571519133040549358:2114] self: [2:7571519133040549357:0] peer: [6:7571519138278423237:0] socket: 627 qp: -1
2025-11-11T17:21:14.318575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519137335516671:2088] [node 6] ICS10 traffic start
2025-11-11T17:21:14.325401Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519133040549360:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.325431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519133040549360:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:14.326523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519137335516677:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45417
2025-11-11T17:21:14.326584Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:14.351157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:14.377094Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519133040549360:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:14.395223Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519137335516660:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:14.393773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519139539851083:2049] [node 3] ICS09 handshake done sender: [1:7571519135244883764:2117] self: [1:7571519135244883761:0] peer: [3:7571519137066208588:0] socket: 645 qp: -1
2025-11-11T17:21:14.396888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519137983537915:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52807
2025-11-11T17:21:14.413689Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519137983537913:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:14.393825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519139539851083:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:14.181651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519132771241192:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.181784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:14.393888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:14.420194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519133688570519:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.418201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519133040549341:2080] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.418352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:14.418408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519133040549341:2080] [node 4] ICP22 created new session: [2:7571519137335516681:2080]
2025-11-11T17:21:14.418455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519137335516681:2080] [node 4] ICS09 handshake done sender: [2:7571519133040549360:2115] self: [2:7571519133040549359:0] peer: [4:7571519140089068582:0] socket: 596 qp: -1
2025-11-11T17:21:14.418497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519137335516681:2080] [node 4] ICS10 traffic start
2025-11-11T17:21:14.418555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:14.419231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519133040549322:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:14.419309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:14.419350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519133040549322:2081] [node 5] ICP22 created new session: [2:7571519137335516683:2081]
2025-11-11T17:21:14.419389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519137335516683:2081] [node 5] ICS09 handshake done sender: [2:7571519137335516660:2117] self: [2:7571519137335516659:0] peer: [5:7571519137983537914:0] socket: 621 qp: -1
2025-11-11T17:21:14.419437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519137335516683:2081] [node 5] ICS10 traffic start
2025-11-11T17:21:14.419469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:14.442454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519133040549329:2088] [node 6] ICP35 event processing took too much time PendingConnection:278(273788613ns) {HandleHandshakeStatus:329(273785221ns) {DropIncomingHandshake:441(1765ns) DropOutgoingHandshake:456(623ns) SwitchToState:178(689ns) ProcessPendingSessionEvents:495(159570ns) {ForwardSessionEventToSession:612(108886ns) ForwardSessionEventToSession:612(3911ns) ForwardSessionEventToSession:612(6010ns) ForwardSessionEventToSession:612(2885ns) ForwardSessionEventToSession:612(6193ns) ForwardSessionEventToSession:612(3066ns) ForwardSessionEventToSession:612(2357ns) ForwardSessionEventToSession:612(4887ns) ForwardSessionEventToSession:612(6258ns) ForwardSessionEventToSession:612(2971ns)}}}
2025-11-11T17:21:14.181840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519132771241192:2049] [node 2] ICP22 created new session: [3:7571519137066208586:2049]
2025-11-11T17:21:14.181890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519137066208586:2049] [node 2] ICS09 handshake done sender: [3:7571519132771241273:2117] self: [3:7571519137066208582:0] peer: [2:7571519133040549326:0] socket: 624 qp: -1
2025-11-11T17:21:14.181930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519137066208586:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:14.181966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:14.228121Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519137066208585:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.228264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519132771241213:2062] [node 5] ICP18 handshake (actor [3:7571519137066208585:2120]) is too old
2025-11-11T17:21:14.279468Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519137066208583:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:14.334257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519132771241191:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:14.358414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519137066208589:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41232
2025-11-11T17:21:14.360121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519132771241191:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(43312343ns) {IncomingHandshake:261(24313296ns) {DropIncomingHandshake:441(47ns) PrepareNewSessionHandshake:476(912ns) {SwitchToState:178(121ns)} GenerateSessionVirtualId:988(160ns) IssueIncomingHandshakeReply:193(63669ns) {SwitchToState:178(631ns)}}}
2025-11-11T17:21:14.365971Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519137066208583:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:14.366261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519132771241191:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:14.366310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:14.366355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519132771241191:2048] [node 1] ICP22 created new session: [3:7571519137066208590:2048]
2025-11-11T17:21:14.366404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519137066208590:2048] [node 1] ICS09 handshake done sender: [3:7571519137066208583:2118] self: [3:7571519137066208588:0] peer: [1:7571519135244883761:0] socket: 646 qp: -1
2025-11-11T17:21:14.366436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519137066208590:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:14.366470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:14.458287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:14.394408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519135244883684:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(16921425ns) {HandleHandshakeStatus:329(16916337ns) {DropIncomingHandshake:441(29297ns) DropOutgoingHandshake:456(415ns) SwitchToState:178(506ns) ProcessPendingSessionEvents:495(119226ns) {ForwardSessionEventToSession:612(62926ns) ForwardSessionEventToSession:612(3095ns) ForwardSessionEventToSession:612(9483ns) ForwardSessionEventToSession:612(3069ns) ForwardSessionEventToSession:612(6411ns) ForwardSessionEventToSession:612(10397ns) ForwardSessionEventToSession:612(3681ns) ForwardSessionEventToSession:612(5367ns) ForwardSessionEventToSession:612(3387ns)}}}
2025-11-11T17:21:14.463777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519133688570519:2049] [node 2] ICP22 created new session: [5:7571519137983537916:2049]
2025-11-11T17:21:14.469270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519137983537916:2049] [node 2] ICS09 handshake done sender: [5:7571519137983537913:2122] self: [5:7571519137983537914:0] peer: [2:7571519137335516659:0] socket: 642 qp: -1
2025-11-11T17:21:14.469346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519137983537916:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:14.469414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:14.469939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519133688570519:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(52695189ns) {HandleHandshakeStatus:329(52691166ns) {DropIncomingHandshake:441(1428ns) DropOutgoingHandshake:456(11764ns) SwitchToState:178(280ns) ProcessPendingSessionEvents:495(113653ns) {ForwardSessionEventToSession:612(75897ns) ForwardSessionEventToSession:612(6536ns) ForwardSessionEventToSession:612(2965ns) ForwardSessionEventToSession:612(2899ns) ForwardSessionEventToSession:612(7470ns) ForwardSessionEventToSession:612(3697ns) ForwardSessionEventToSession:612(5622ns)}}}
2025-11-11T17:21:14.550221Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:14.550473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519140089068580:2048] [node 1] ICS01 socket: 607 reason# EndOfStream
2025-11-11T17:21:14.550516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519135794101184:2048] [node 1] ICP30 unregister session Session# [4:7571519140089068580:2048] VirtualId# [4:7571519140089068575:0]
2025-11-11T17:21:14.550555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519140089068580:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.550635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:14.550676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:14.546392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:14.546597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519137335516673:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-11-11T17:21:14.546643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519133040549283:2048] [node 1] ICP30 unregister session Session# [2:7571519137335516673:2048] VirtualId# [2:7571519137335516667:0]
2025-11-11T17:21:14.546680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519137335516673:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.546763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:14.546802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:14.552801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:14.553055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519137066208590:2048] [node 1] ICS01 socket: 646 reason# EndOfStream
2025-11-11T17:21:14.553109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519132771241191:2048] [node 1] ICP30 unregister session Session# [3:7571519137066208590:2048] VirtualId# [3:7571519137066208588:0]
2025-11-11T17:21:14.553148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519137066208590:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.553428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:14.553495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:14.557772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:14.558062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519137983537909:2048] [node 1] ICS01 socket: 611 reason# EndOfStream
2025-11-11T17:21:14.558107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519133688570518:2048] [node 1] ICP30 unregister session Session# [5:7571519137983537909:2048] VirtualId# [5:7571519137983537900:0]
2025-11-11T17:21:14.558140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519137983537909:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.558239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:14.558282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:14.558354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:14.564391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519138278423247:2048] [node 1] ICS01 socket: 604 reason# EndOfStream
2025-11-11T17:21:14.564447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519133983455848:2048] [node 1] ICP30 unregister session Session# [6:7571519138278423247:2048] VirtualId# [6:7571519138278423232:0]
2025-11-11T17:21:14.564487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519138278423247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.564597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:14.564641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:14.588114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:14.588326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519137066208586:2049] [node 2] ICS01 socket: 624 reason# EndOfStream
2025-11-11T17:21:14.588367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519132771241192:2049] [node 2] ICP30 unregister session Session# [3:7571519137066208586:2049] VirtualId# [3:7571519137066208582:0]
2025-11-11T17:21:14.588400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519137066208586:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.588475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:14.588519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:14.597096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:14.597337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519140089068584:2049] [node 2] ICS01 socket: 609 reason# EndOfStream
2025-11-11T17:21:14.597388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519135794101185:2049] [node 2] ICP30 unregister session Session# [4:7571519140089068584:2049] VirtualId# [4:7571519140089068582:0]
2025-11-11T17:21:14.597427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519140089068584:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.597519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:14.597570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:14.597679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:14.597867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519137983537916:2049] [node 2] ICS01 socket: 642 reason# EndOfStream
2025-11-11T17:21:14.597899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519133688570519:2049] [node 2] ICP30 unregister session Session# [5:7571519137983537916:2049] VirtualId# [5:7571519137983537914:0]
2025-11-11T17:21:14.597932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519137983537916:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.597979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:14.598022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:14.598099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:14.598270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519138278423245:2049] [node 2] ICS01 socket: 629 reason# EndOfStream
2025-11-11T17:21:14.598320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519133983455849:2049] [node 2] ICP30 unregister session Session# [6:7571519138278423245:2049] VirtualId# [6:7571519138278423237:0]
2025-11-11T17:21:14.598345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519138278423245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.598397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:14.598430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:14.681799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:14.682006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519140089068574:2050] [node 3] ICS01 socket: 606 reason# EndOfStream
2025-11-11T17:21:14.682057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519135794101186:2050] [node 3] ICP30 unregister session Session# [4:7571519140089068574:2050] VirtualId# [4:7571519135794101269:0]
2025-11-11T17:21:14.681796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:14.682006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519138278423233:2050] [node 3] ICS01 socket: 605 reason# EndOfStream
2025-11-11T17:21:14.682086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519140089068574:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.682051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519133983455850:2050] [node 3] ICP30 unregister session Session# [6:7571519138278423233:2050] VirtualId# [6:7571519133983455928:0]
2025-11-11T17:21:14.682166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:14.682085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519138278423233:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.682208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:14.682159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:14.682199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:14.682600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:14.682863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519137983537911:2050] [node 3] ICS01 socket: 614 reason# EndOfStream
2025-11-11T17:21:14.682913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519133688570520:2050] [node 3] ICP30 unregister session Session# [5:7571519137983537911:2050] VirtualId# [5:7571519133688570602:0]
2025-11-11T17:21:14.682959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519137983537911:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.683045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:14.683104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:14.737761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:14.737756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:14.738095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519137983537904:2073] [node 4] ICS01 socket: 595 reason# EndOfStream
2025-11-11T17:21:14.738045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519138278423242:2090] [node 4] ICS01 socket: 616 reason# EndOfStream
2025-11-11T17:21:14.738135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519133688570548:2073] [node 4] ICP30 unregister session Session# [5:7571519137983537904:2073] VirtualId# [5:7571519133688570596:0]
2025-11-11T17:21:14.738098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519133983455901:2090] [node 4] ICP30 unregister session Session# [6:7571519138278423242:2090] VirtualId# [6:7571519133983455932:0]
2025-11-11T17:21:14.738163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519137983537904:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.738138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519138278423242:2090] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.738247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:14.738235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:14.738288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:14.738283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:14.785849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:14.786102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519138278423235:2088] [node 5] ICS01 socket: 591 reason# EndOfStream
2025-11-11T17:21:14.786145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519133983455917:2088] [node 5] ICP30 unregister session Session# [6:7571519138278423235:2088] VirtualId# [6:7571519133983455927:0]
2025-11-11T17:21:14.786190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519138278423235:2088] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:14.786275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:14.786320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:14.883785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519140492980672:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:14.883827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519140492980673:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:14.925911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519138425978002:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:14.925970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519138425978003:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:14.961875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519138116304099:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:14.961923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519138116304100:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:14.975571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519139785404577:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:14.975612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519139785404578:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:14.975638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519139785404579:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:15.007655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519141913235914:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:15.007707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519141913235915:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:15.007739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519141913235916:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:15.034013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519142330765808:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:15.034056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519142330765809:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:15.034085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519142330765810:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:15.373359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519142720945335:2081] [node 5] ICP01 ready to work
2025-11-11T17:21:15.426605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519142411271442:2070] [node 4] ICP01 ready to work
2025-11-11T17:21:15.454115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519144080371911:2066] [node 6] ICP01 ready to work
2025-11-11T17:21:15.437773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519141913235936:2068] [node 4] ICP01 ready to work
2025-11-11T17:21:15.429998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519144787948031:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:15.465805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519144787948032:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:15.463131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519142411271444:2072] [node 6] ICP01 ready to work
2025-11-11T17:21:15.514794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519142330765822:2060] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.533664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519142720945355:2085] [node 6] ICP01 ready to work
2025-11-11T17:21:15.505670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519144080371910:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.507834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519141913235940:2065] [node 6] ICP01 ready to work
2025-11-11T17:21:15.533925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519144787948033:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:15.505762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519142411271448:2080] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.567761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519144080371935:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23074
2025-11-11T17:21:15.551989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519142330765810:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(101921234ns) {Configure:104(101915517ns) {ProcessConfigured:135(102034ns) {StartInitialHandshake:157(98937ns) {DropHandshakes:469(3250ns) {DropIncomingHandshake:441(516ns) DropOutgoingHandshake:456(415ns)} GenerateSessionVirtualId:988(1431ns) PrepareNewSessionHandshake:476(1705ns) {SwitchToState:178(570ns)}}}}}
2025-11-11T17:21:15.624215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519141913235915:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(186269961ns) {Configure:104(186264929ns) {ProcessConfigured:135(78776ns) {StartInitialHandshake:157(76224ns) {DropHandshakes:469(2903ns) {DropIncomingHandshake:441(811ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(853ns) PrepareNewSessionHandshake:476(2548ns) {SwitchToState:178(912ns)}}}}}
2025-11-11T17:21:15.482429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519142720945356:2083] [node 4] ICP01 ready to work
2025-11-11T17:21:15.725044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519138116304099:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(277044805ns) {Configure:104(277038864ns) {ProcessConfigured:135(73316ns) {StartInitialHandshake:157(71061ns) {DropHandshakes:469(3416ns) {DropIncomingHandshake:441(683ns) DropOutgoingHandshake:456(632ns)} GenerateSessionVirtualId:988(928ns) PrepareNewSessionHandshake:476(3809ns) {SwitchToState:178(508ns)}}}}}
2025-11-11T17:21:15.638044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519142330765840:2071] [node 5] ICP01 ready to work
2025-11-11T17:21:15.698496Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519144080371935:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.743188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519138425978003:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(289806948ns) {Configure:104(289798920ns) {ProcessConfigured:135(83191ns) {StartInitialHandshake:157(76323ns) {DropHandshakes:469(2104ns) {DropIncomingHandshake:441(532ns) DropOutgoingHandshake:456(410ns)} GenerateSessionVirtualId:988(933ns) PrepareNewSessionHandshake:476(1855ns) {SwitchToState:178(598ns)}}}}}
2025-11-11T17:21:15.634966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519141913235977:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.688577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519142330765869:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:15.739236Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519142411271442:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(278649214ns) {Configure:104(278638990ns) {ProcessConfigured:135(75068ns) {StartInitialHandshake:157(71771ns) {DropHandshakes:469(3221ns) {DropIncomingHandshake:441(777ns) DropOutgoingHandshake:456(925ns)} GenerateSessionVirtualId:988(1268ns) PrepareNewSessionHandshake:476(1545ns) {SwitchToState:178(512ns)}}}}}
2025-11-11T17:21:15.747854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519144787948032:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(150007529ns) {Configure:104(150001010ns) {ProcessConfigured:135(70068ns) {StartInitialHandshake:157(66785ns) {DropHandshakes:469(3310ns) {DropIncomingHandshake:441(780ns) DropOutgoingHandshake:456(727ns)} GenerateSessionVirtualId:988(859ns) PrepareNewSessionHandshake:476(2208ns) {SwitchToState:178(788ns)}}}}}
2025-11-11T17:21:15.757169Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519141913235976:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.784624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519142720945355:2085] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(41153270ns) {Configure:104(41148866ns) {ProcessConfigured:135(98402ns) {StartInitialHandshake:157(95871ns) {DropHandshakes:469(2547ns) {DropIncomingHandshake:441(598ns) DropOutgoingHandshake:456(390ns)} GenerateSessionVirtualId:988(625ns) PrepareNewSessionHandshake:476(1341ns) {SwitchToState:178(477ns)}}}}}
2025-11-11T17:21:15.785375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519142720945368:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55
2025-11-11T17:21:15.785538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519142720945369:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55
2025-11-11T17:21:15.755126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519142411271444:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(29808606ns) {Configure:104(29803625ns) {ProcessConfigured:135(75120ns) {StartInitialHandshake:157(72410ns) {DropHandshakes:469(3922ns) {DropIncomingHandshake:441(826ns) DropOutgoingHandshake:456(738ns)} GenerateSessionVirtualId:988(1093ns) PrepareNewSessionHandshake:476(3533ns) {SwitchToState:178(1444ns)}}}}}
2025-11-11T17:21:15.688626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519142330765870:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:15.796434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519142720945335:2081] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(422717356ns) {Configure:104(422712531ns) {ProcessConfigured:135(8502024ns) {StartInitialHandshake:157(8498785ns) {DropHandshakes:469(2459ns) {DropIncomingHandshake:441(624ns) DropOutgoingHandshake:456(496ns)} GenerateSessionVirtualId:988(694ns) PrepareNewSessionHandshake:476(3544ns) {SwitchToState:178(1585ns)}}}}}
2025-11-11T17:21:15.757238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519141913235976:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:15.796599Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519144080371937:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.796649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519144080371937:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.755794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519142411271467:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51311
2025-11-11T17:21:15.720045Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519142330765870:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.778634Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519141913235977:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.726273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519142330765840:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.824144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519144787948038:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11614
2025-11-11T17:21:15.828496Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945368:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.782397Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271467:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.828652Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519144080371937:2109] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:15.789815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519141913235976:2105] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.824209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519144787948039:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11614
2025-11-11T17:21:15.858331Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945369:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.745144Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519142330765869:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.802736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519138116304100:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(296784450ns) {Configure:104(296777763ns) {ProcessConfigured:135(77605ns) {StartInitialHandshake:157(74966ns) {DropHandshakes:469(3227ns) {DropIncomingHandshake:441(819ns) DropOutgoingHandshake:456(928ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:476(1525ns) {SwitchToState:178(654ns)}}}}}
2025-11-11T17:21:15.878344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519144080371911:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:15.826476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519140492980673:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(391329145ns) {Configure:104(391321306ns) {ProcessConfigured:135(58184ns) {StartInitialHandshake:157(55921ns) {DropHandshakes:469(983ns) {DropIncomingHandshake:441(216ns) DropOutgoingHandshake:456(169ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(2104ns) {SwitchToState:178(815ns)}}}}}
2025-11-11T17:21:15.822625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519141913235914:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(392600402ns) {Configure:104(392594899ns) {ProcessConfigured:135(24113782ns) {StartInitialHandshake:157(24110006ns) {DropHandshakes:469(1687ns) {DropIncomingHandshake:441(497ns) DropOutgoingHandshake:456(205ns)} GenerateSessionVirtualId:988(789ns) PrepareNewSessionHandshake:476(4963ns) {SwitchToState:178(1939ns)}}}}}
2025-11-11T17:21:15.845679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519142411271472:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51311
2025-11-11T17:21:15.775597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519142330765822:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.862851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519138425978002:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(320034906ns) {Configure:104(320024730ns) {ProcessConfigured:135(53944ns) {StartInitialHandshake:157(52068ns) {DropHandshakes:469(1379ns) {DropIncomingHandshake:441(322ns) DropOutgoingHandshake:456(340ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:476(1349ns) {SwitchToState:178(361ns)}}}}}
2025-11-11T17:21:15.878433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:15.778044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519142330765888:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:15.779369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519142330765889:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:15.856510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519142330765891:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:15.875726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519142330765808:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(293962786ns) {Configure:104(293956782ns) {ProcessConfigured:135(31422602ns) {StartInitialHandshake:157(31419964ns) {DropHandshakes:469(2472ns) {DropIncomingHandshake:441(497ns) DropOutgoingHandshake:456(625ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:476(2908ns) {SwitchToState:178(579ns)}}}}}
2025-11-11T17:21:15.879991Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519142330765870:2101] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:15.881739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519142330765893:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:15.891551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519142330765840:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:15.891624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:15.891671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519142330765840:2071] [node 5] ICP22 created new session: [6:7571519142330765894:2071]
2025-11-11T17:21:15.891721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519142330765894:2071] [node 5] ICS09 handshake done sender: [6:7571519142330765870:2101] self: [6:7571519142330765882:0] peer: [5:7571519141913235975:0] socket: 604 qp: -1
2025-11-11T17:21:15.891754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519142330765894:2071] [node 5] ICS10 traffic start
2025-11-11T17:21:15.891787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:15.892412Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8794) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:15.894336Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519142330765888:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.899398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519142330765822:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8794) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519142330765869:2100] held: no
2025-11-11T17:21:15.902954Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519142330765869:2100] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:15.919237Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519142330765891:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.919375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519142330765810:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.919620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519142330765809:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.881370Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948039:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.884703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948038:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.927742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519140492980672:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(420948385ns) {Configure:104(420936895ns) {ProcessConfigured:135(13079344ns) {StartInitialHandshake:157(13076422ns) {DropHandshakes:469(2472ns) {DropIncomingHandshake:441(532ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(1412ns) PrepareNewSessionHandshake:476(5029ns) {SwitchToState:178(2151ns)}}}}}
2025-11-11T17:21:15.932337Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948043:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.932364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519144787948043:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:15.933337Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519144787948043:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:15.938579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519144787948031:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(340581747ns) {Configure:104(340575848ns) {ProcessConfigured:135(34995407ns) {StartInitialHandshake:157(34991564ns) {DropHandshakes:469(2422ns) {DropIncomingHandshake:441(466ns) DropOutgoingHandshake:456(495ns)} GenerateSessionVirtualId:988(722ns) PrepareNewSessionHandshake:476(3313ns) {SwitchToState:178(1228ns)}}}}}
2025-11-11T17:21:15.851838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519142411271474:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51311
2025-11-11T17:21:15.937847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519142330765822:2060] [node 4] ICP35 event processing took too much time PendingConnection:278(16078492ns) {HandleHandshakeStatus:395(16075233ns) {LogHandshakeFail:472(12514ns) {UpdateErrorStateLog:544(4626ns)} DropOutgoingHandshake:456(2115ns) ProcessPendingSessionEvents:495(11028944ns) {DropSessionEvent:512(4162495ns) DropSessionEvent:512(11665ns) DropSessionEvent:512(6331ns) DropSessionEvent:512(23181ns) DropSessionEvent:512(6201ns) DropSessionEvent:512(7314ns) DropSessionEvent:512(7122ns) DropSessionEvent:512(4955ns) DropSessionEvent:512(5289ns) DropSessionEvent:512(4758ns)}}}
2025-11-11T17:21:15.837852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519141913235940:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:15.863666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519142720945377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55
2025-11-11T17:21:15.878485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519144080371911:2066] [node 6] ICP22 created new session: [4:7571519144080371952:2066]
2025-11-11T17:21:15.937944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519142330765822:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:15.937995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:15.941789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519142330765822:2060] [node 4] ICP22 created new session: [6:7571519142330765898:2060]
2025-11-11T17:21:15.947155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519142330765898:2060] [node 4] ICS09 handshake done sender: [6:7571519142330765869:2100] self: [6:7571519142330765887:0] peer: [4:7571519144080371936:0] socket: 603 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.877906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519142411271476:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51311
2025-11-11T17:21:15.880197Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945377:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.891346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519142720945381:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55
2025-11-11T17:21:15.913699Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945365:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.913734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519142720945365:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:15.918261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519142720945356:2083] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(217052035ns) {Configure:104(217044593ns) {ProcessConfigured:135(13868397ns) {StartInitialHandshake:157(13864984ns) {DropHandshakes:469(6058898ns) {DropIncomingHandshake:441(1066ns) DropOutgoingHandshake:456(331ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:476(2480ns) {SwitchToState:178(813ns)}}}}}
2025-11-11T17:21:15.918514Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519142720945365:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:15.934094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519138425978003:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:15.934184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:15.934235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519138425978003:2049] [node 3] ICP22 created new session: [2:7571519142720945385:2049]
2025-11-11T17:21:15.934306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519142720945385:2049] [node 3] ICS09 handshake done sender: [2:7571519142720945365:2110] self: [2:7571519142720945364:0] peer: [3:7571519142411271478:0] socket: 613 qp: -1
2025-11-11T17:21:15.934343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519142720945385:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:15.934375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:15.938773Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945367:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.938813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519142720945367:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:15.944636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945381:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.837922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:15.944127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519140492980673:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:15.878539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519144080371952:2066] [node 6] ICS09 handshake done sender: [4:7571519144080371937:2109] self: [4:7571519144080371936:0] peer: [6:7571519142330765887:0] socket: 597 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.950766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:15.899397Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271472:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.837980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519141913235940:2065] [node 6] ICP22 created new session: [5:7571519141913235990:2065]
2025-11-11T17:21:15.955359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519142330765900:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:15.970032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519142720945367:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:15.878576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519144080371952:2066] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.957719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519142330765898:2060] [node 4] ICS10 traffic start
2025-11-11T17:21:15.838038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519141913235990:2065] [node 6] ICS09 handshake done sender: [5:7571519141913235976:2105] self: [5:7571519141913235975:0] peer: [6:7571519142330765882:0] socket: 602 qp: -1
2025-11-11T17:21:15.904097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271476:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.965939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519140492980673:2049] [node 3] ICP22 created new session: [1:7571519144787948052:2049]
2025-11-11T17:21:15.878637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:15.971107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519142720945355:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:15.969688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:15.980769Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:16.006181Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519142330765891:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:16.024966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519146625733198:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:16.025736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519142330765809:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519142330765891:2118] held: no
2025-11-11T17:21:16.034469Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519142330765888:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:15.905832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519138116304100:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.978084Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948037:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.971177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:15.882277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519144080371951:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23074
2025-11-11T17:21:15.883441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519139785404578:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(350472479ns) {Configure:104(350460195ns) {ProcessConfigured:135(16539153ns) {StartInitialHandshake:157(16536481ns) {DropHandshakes:469(2122ns) {DropIncomingHandshake:441(539ns) DropOutgoingHandshake:456(470ns)} GenerateSessionVirtualId:988(364ns) PrepareNewSessionHandshake:476(3013ns) {SwitchToState:178(1264ns)}}}}}
2025-11-11T17:21:15.887252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519139785404579:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(397056164ns) {Configure:104(397050026ns) {ProcessConfigured:135(22253417ns) {StartInitialHandshake:157(22250437ns) {DropHandshakes:469(1761ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(251ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(5232ns) {SwitchToState:178(2271ns)}}}}}
2025-11-11T17:21:15.898192Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519144080371951:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.941397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519139785404577:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(421230948ns) {Configure:104(421217198ns) {ProcessConfigured:135(25531151ns) {StartInitialHandshake:157(25527561ns) {DropHandshakes:469(2765ns) {DropIncomingHandshake:441(952ns) DropOutgoingHandshake:456(396ns)} GenerateSessionVirtualId:988(1192ns) PrepareNewSessionHandshake:476(4710ns) {SwitchToState:178(1840ns)}}}}}
2025-11-11T17:21:15.945929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519139785404577:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.969645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519144080371939:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.969704Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519144080371939:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:15.994269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519144080371960:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23074
2025-11-11T17:21:15.994334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519144080371961:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23074
2025-11-11T17:21:16.000597Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519144080371939:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:16.022520Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519144080371951:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:16.034664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519144080371910:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.034713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:16.034821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519144080371960:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.034911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519139785404577:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.042588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519139785404578:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:16.045355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:16.049828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519144080371910:2067] [node 5] ICP22 created new session: [4:7571519148375339258:2067]
2025-11-11T17:21:16.065322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519139785404577:2048] [node 1] ICP22 created new session: [4:7571519148375339261:2048]
2025-11-11T17:21:16.065392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519148375339261:2048] [node 1] ICS09 handshake done sender: [4:7571519144080371951:2114] self: [4:7571519144080371958:0] peer: [1:7571519144787948036:0] socket: 633 qp: -1
2025-11-11T17:21:16.065437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519148375339261:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:16.065538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:16.066751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519148375339258:2067] [node 5] ICS09 handshake done sender: [4:7571519144080371939:2110] self: [4:7571519144080371938:0] peer: [5:7571519141913236001:0] socket: 598 qp: -1
2025-11-11T17:21:16.073012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519148375339258:2067] [node 5] ICS10 traffic start
2025-11-11T17:21:15.838080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519141913235990:2065] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.910437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519142411271477:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51311
2025-11-11T17:21:16.077896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:15.971225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519142720945355:2085] [node 6] ICP22 created new session: [2:7571519142720945390:2085]
2025-11-11T17:21:15.978115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519144787948037:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:16.071655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519142330765822:2060] [node 4] ICP35 event processing took too much time PendingConnection:278(96840798ns) {HandleHandshakeStatus:329(96838020ns) {DropIncomingHandshake:441(837ns) DropOutgoingHandshake:456(462ns) SwitchToState:178(488ns) ProcessPendingSessionEvents:495(881ns)}}
2025-11-11T17:21:15.838122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.916464Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271474:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.978672Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948035:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.840668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519141913235992:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
2025-11-11T17:21:15.864026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519141913235916:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(380786975ns) {Configure:104(380781331ns) {ProcessConfigured:135(7127987ns) {StartInitialHandshake:157(7125116ns) {DropHandshakes:469(3310ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(799ns)} GenerateSessionVirtualId:988(553ns) PrepareNewSessionHandshake:476(1566ns) {SwitchToState:178(264ns)}}}}}
2025-11-11T17:21:15.887341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519141913235996:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
2025-11-11T17:21:15.914727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519141913235992:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.915087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519141913235936:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(336891602ns) {Configure:104(336884416ns) {ProcessConfigured:135(55340ns) {StartInitialHandshake:157(52633ns) {DropHandshakes:469(1727ns) {DropIncomingHandshake:441(484ns) DropOutgoingHandshake:456(281ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:476(1760ns) {SwitchToState:178(424ns)}}}}}
2025-11-11T17:21:15.917882Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519141913235996:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.944126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519141913235915:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.945788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519141913235914:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.949696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519141913235936:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:16.002372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519146208203298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
2025-11-11T17:21:16.002526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519146208203299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
2025-11-11T17:21:16.008821Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519141913235992:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:16.008994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519141913235977:2106] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:16.046564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519141913235936:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.046610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:16.046668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519141913235936:2068] [node 4] ICP22 created new session: [5:7571519146208203300:2068]
2025-11-11T17:21:16.046712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519146208203300:2068] [node 4] ICS09 handshake done sender: [5:7571519141913235977:2106] self: [5:7571519141913236001:0] peer: [4:7571519144080371938:0] socket: 600 qp: -1
2025-11-11T17:21:16.046746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519146208203300:2068] [node 4] ICS10 traffic start
2025-11-11T17:21:16.046774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:16.047108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519141913235914:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.047151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:16.047188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519141913235914:2048] [node 1] ICP22 created new session: [5:7571519146208203302:2048]
2025-11-11T17:21:16.047231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519146208203302:2048] [node 1] ICS09 handshake done sender: [5:7571519141913235992:2114] self: [5:7571519141913235999:0] peer: [1:7571519144787948034:0] socket: 623 qp: -1
2025-11-11T17:21:16.047261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519146208203302:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:16.047291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:16.080767Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:16.081506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519141913235915:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519141913235996:2116] held: no
2025-11-11T17:21:16.084443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519146208203305:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
2025-11-11T17:21:16.095767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519146208203305:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.115104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519141913235916:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:15.971278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519142720945390:2085] [node 6] ICS09 handshake done sender: [2:7571519142720945367:2111] self: [2:7571519142720945366:0] peer: [6:7571519142330765897:0] socket: 614 qp: -1
2025-11-11T17:21:16.071727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519142330765810:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.083136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:16.098946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519139785404577:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(63816761ns) {HandleHandshakeStatus:329(63814692ns) {DropIncomingHandshake:441(601ns) DropOutgoingHandshake:456(4333ns) SwitchToState:178(322ns) ProcessPendingSessionEvents:495(8720189ns) {ForwardSessionEventToSession:612(8661719ns) ForwardSessionEventToSession:612(8819ns) ForwardSessionEventToSession:612(3838ns) ForwardSessionEventToSession:612(8873ns) ForwardSessionEventToSession:612(3205ns) ForwardSessionEventToSession:612(3296ns) ForwardSessionEventToSession:612(6421ns) ForwardSessionEventToSession:612(5598ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(5288ns)}}}
2025-11-11T17:21:16.114310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519148375339265:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23074
2025-11-11T17:21:16.114872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519144080371910:2067] [node 5] ICP35 event processing took too much time PendingConnection:278(80002923ns) {HandleHandshakeStatus:329(80000697ns) {DropIncomingHandshake:441(306ns) DropOutgoingHandshake:456(922ns) SwitchToState:178(137ns) ProcessPendingSessionEvents:495(19024729ns) {ForwardSessionEventToSession:612(18965023ns) ForwardSessionEventToSession:612(9052ns) ForwardSessionEventToSession:612(2969ns) ForwardSessionEventToSession:612(8873ns) ForwardSessionEventToSession:612(2702ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(4346ns) ForwardSessionEventToSession:612(3349ns) ForwardSessionEventToSession:612(5245ns) ForwardSessionEventToSession:612(5263ns)}}}
2025-11-11T17:21:15.931292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519138116304099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:16.090678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519142330765810:2050] [node 3] ICP22 created new session: [6:7571519146625733200:2050]
2025-11-11T17:21:16.102064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519142330765809:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(33121659ns) {HandleHandshakeStatus:395(33117163ns) {LogHandshakeFail:472(16537ns) {UpdateErrorStateLog:544(6390ns)} DropOutgoingHandshake:456(2880ns) ProcessPendingSessionEvents:495(11957105ns) {DropSessionEvent:512(10768ns) DropSessionEvent:512(4845ns) DropSessionEvent:512(5336ns) DropSessionEvent:512(11888985ns) DropSessionEvent:512(6703ns) DropSessionEvent:512(5872ns) DropSessionEvent:512(6969ns) DropSessionEvent:512(5943ns) DropSessionEvent:512(7181ns) DropSessionEvent:512(5885ns)}}}
2025-11-11T17:21:16.102132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519142330765809:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.102283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519146625733200:2050] [node 3] ICS09 handshake done sender: [6:7571519142330765888:2116] self: [6:7571519142330765896:0] peer: [3:7571519142411271465:0] socket: 608 qp: -1
2025-11-11T17:21:16.102325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519146625733200:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:16.102372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:16.102899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519142330765810:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(30918370ns) {HandleHandshakeStatus:329(30916615ns) {DropIncomingHandshake:441(712ns) DropOutgoingHandshake:456(5808ns) SwitchToState:178(720ns) ProcessPendingSessionEvents:495(118559ns) {ForwardSessionEventToSession:612(64613ns) ForwardSessionEventToSession:612(12316ns) ForwardSessionEventToSession:612(2809ns) ForwardSessionEventToSession:612(4836ns) ForwardSessionEventToSession:612(2512ns) ForwardSessionEventToSession:612(2930ns) ForwardSessionEventToSession:612(5510ns) ForwardSessionEventToSession:612(6257ns) ForwardSessionEventToSession:612(3807ns) ForwardSessionEventToSession:612(5274ns)}}}
2025-11-11T17:21:16.104103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:16.115245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519142330765809:2049] [node 2] ICP22 created new session: [6:7571519146625733202:2049]
2025-11-11T17:21:16.115308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519146625733202:2049] [node 2] ICS09 handshake done sender: [6:7571519142330765891:2118] self: [6:7571519142330765897:0] peer: [2:7571519142720945366:0] socket: 616 qp: -1
2025-11-11T17:21:16.115350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519146625733202:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:16.115410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:16.117713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519146625733199:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:16.145799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519144080371960:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:16.146102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519139785404578:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.146168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:16.146252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519139785404578:2049] [node 2] ICP22 created new session: [4:7571519148375339266:2049]
2025-11-11T17:21:16.146308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519148375339266:2049] [node 2] ICS09 handshake done sender: [4:7571519144080371960:2117] self: [4:7571519148375339260:0] peer: [2:7571519142720945382:0] socket: 645 qp: -1
2025-11-11T17:21:16.146352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519148375339266:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:16.146409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:15.971317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519142720945390:2085] [node 6] ICS10 traffic start
2025-11-11T17:21:15.978693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519144787948035:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.971351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:15.980028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519144787948053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11614
2025-11-11T17:21:15.931504Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271477:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.171854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519146208203307:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
2025-11-11T17:21:16.174223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519146208203304:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11507
2025-11-11T17:21:15.931971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519142411271481:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51311
2025-11-11T17:21:15.980155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519144787948054:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11614
2025-11-11T17:21:15.987364Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945373:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.173957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519148375339262:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23074
2025-11-11T17:21:16.169680Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519146625733199:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.938887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519142411271448:2080] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(229577185ns) {Configure:104(229569673ns) {ProcessConfigured:135(98203ns) {StartInitialHandshake:157(94611ns) {DropHandshakes:469(6799ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(390ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:476(2577ns) {SwitchToState:178(1104ns)}}}}}
2025-11-11T17:21:15.987412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519142720945373:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:15.948275Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271466:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.948330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519142411271466:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:15.950248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519142411271484:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51311
2025-11-11T17:21:16.005863Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519142411271466:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:16.006451Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:16.008566Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:16.018151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519138116304100:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519142411271472:2115] held: no
2025-11-11T17:21:16.018283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519138116304099:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519142411271474:2116] held: no
2025-11-11T17:21:16.018409Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519142411271472:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:16.024498Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519142411271474:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:16.042983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519138116304100:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.043059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:16.043112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519138116304100:2049] [node 2] ICP22 created new session: [3:7571519146706238783:2049]
2025-11-11T17:21:16.043162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519146706238783:2049] [node 2] ICS09 handshake done sender: [3:7571519142411271472:2115] self: [3:7571519142411271478:0] peer: [2:7571519142720945364:0] socket: 615 qp: -1
2025-11-11T17:21:16.043202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519146706238783:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:16.043237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:16.043629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519142411271444:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.043709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:16.043749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519142411271444:2072] [node 6] ICP22 created new session: [3:7571519146706238785:2072]
2025-11-11T17:21:16.043798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519146706238785:2072] [node 6] ICS09 handshake done sender: [3:7571519142411271466:2112] self: [3:7571519142411271465:0] peer: [6:7571519142330765896:0] socket: 607 qp: -1
2025-11-11T17:21:16.043838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519146706238785:2072] [node 6] ICS10 traffic start
2025-11-11T17:21:16.043866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:16.044225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519138116304099:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.044294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:16.044331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519138116304099:2048] [node 1] ICP22 created new session: [3:7571519146706238787:2048]
2025-11-11T17:21:16.044378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519146706238787:2048] [node 1] ICS09 handshake done sender: [3:7571519142411271474:2116] self: [3:7571519142411271482:0] peer: [1:7571519144787948042:0] socket: 626 qp: -1
2025-11-11T17:21:16.044403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519146706238787:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:16.044432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:16.117392Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271480:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.117418Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519142411271480:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:16.172770Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519142411271480:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:16.197692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519142411271448:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.197830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:16.201637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519142411271448:2080] [node 5] ICP22 created new session: [3:7571519146706238792:2080]
2025-11-11T17:21:16.169914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519142330765808:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:16.189730Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519146208203305:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:16.198088Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519148375339262:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:15.987375Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519144787948037:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:16.033901Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519142720945373:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:16.072829Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519142720945384:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.072872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519142720945384:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:16.076076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519142720945335:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.076153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:16.076224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519142720945335:2081] [node 5] ICP22 created new session: [2:7571519147015912691:2081]
2025-11-11T17:21:16.076285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519147015912691:2081] [node 5] ICS09 handshake done sender: [2:7571519142720945373:2114] self: [2:7571519142720945372:0] peer: [5:7571519141913236000:0] socket: 629 qp: -1
2025-11-11T17:21:16.076321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519147015912691:2081] [node 5] ICS10 traffic start
2025-11-11T17:21:16.076364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:16.109818Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519142720945384:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:16.130673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519142720945356:2083] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.140337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:16.153917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519147015912690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55
2025-11-11T17:21:16.160768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519142720945356:2083] [node 4] ICP22 created new session: [2:7571519147015912694:2083]
2025-11-11T17:21:16.162928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519147015912694:2083] [node 4] ICS09 handshake done sender: [2:7571519142720945384:2118] self: [2:7571519142720945382:0] peer: [4:7571519148375339260:0] socket: 644 qp: -1
2025-11-11T17:21:16.169717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519147015912694:2083] [node 4] ICS10 traffic start
2025-11-11T17:21:16.169798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:16.170256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519142720945356:2083] [node 4] ICP35 event processing took too much time PendingConnection:278(39404962ns) {HandleHandshakeStatus:329(39401371ns) {DropIncomingHandshake:441(16346ns) DropOutgoingHandshake:456(381ns) SwitchToState:178(508ns) ProcessPendingSessionEvents:495(112466ns) {ForwardSessionEventToSession:612(64350ns) ForwardSessionEventToSession:612(3432ns) ForwardSessionEventToSession:612(3032ns) ForwardSessionEventToSession:612(5576ns) ForwardSessionEventToSession:612(2587ns) ForwardSessionEventToSession:612(6960ns) ForwardSessionEventToSession:612(2710ns) ForwardSessionEventToSession:612(3263ns) ForwardSessionEventToSession:612(8057ns) ForwardSessionEventToSession:612(3795ns)}}}
2025-11-11T17:21:16.174309Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519147015912690:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.188801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519138425978002:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:16.217689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519146706238792:2080] [node 5] ICS09 handshake done sender: [3:7571519142411271480:2119] self: [3:7571519142411271479:0] peer: [5:7571519146208203306:0] socket: 655 qp: -1
2025-11-11T17:21:16.213741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519141913235916:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:15.998194Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519144787948035:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:16.229699Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519141913235996:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:16.242373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:16.007784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519144787948052:2049] [node 3] ICS09 handshake done sender: [1:7571519144787948043:2114] self: [1:7571519144787948042:0] peer: [3:7571519142411271482:0] socket: 624 qp: -1
2025-11-11T17:21:16.242447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519141913235916:2050] [node 3] ICP22 created new session: [5:7571519146208203308:2050]
2025-11-11T17:21:16.242523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519146208203308:2050] [node 3] ICS09 handshake done sender: [5:7571519146208203305:2121] self: [5:7571519146208203306:0] peer: [3:7571519142411271479:0] socket: 657 qp: -1
2025-11-11T17:21:16.007550Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948053:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.242566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519146208203308:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:16.214079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519139785404579:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:16.021827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519144787948052:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:16.242617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:16.021898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:16.022792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519144787948032:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.243146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519141913235916:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(29193435ns) {HandleHandshakeStatus:329(29189156ns) {DropIncomingHandshake:441(4073ns) DropOutgoingHandshake:456(29121ns) SwitchToState:178(684ns) ProcessPendingSessionEvents:495(102404ns) {ForwardSessionEventToSession:612(54672ns) ForwardSessionEventToSession:612(2990ns) ForwardSessionEventToSession:612(8158ns) ForwardSessionEventToSession:612(2957ns) ForwardSessionEventToSession:612(3618ns) ForwardSessionEventToSession:612(5554ns) ForwardSessionEventToSession:612(3793ns) ForwardSessionEventToSession:612(7880ns) ForwardSessionEventToSession:612(2492ns) ForwardSessionEventToSession:612(3050ns)}}}
2025-11-11T17:21:16.263400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519141913235915:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.022863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:16.263505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:16.022921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519144787948032:2108] [node 5] ICP22 created new session: [1:7571519149082915355:2108]
2025-11-11T17:21:16.263557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519141913235915:2049] [node 2] ICP22 created new session: [5:7571519146208203310:2049]
2025-11-11T17:21:16.022975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519149082915355:2108] [node 5] ICS09 handshake done sender: [1:7571519144787948035:2110] self: [1:7571519144787948034:0] peer: [5:7571519141913235999:0] socket: 621 qp: -1
2025-11-11T17:21:16.263609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519146208203310:2049] [node 2] ICS09 handshake done sender: [5:7571519141913235996:2116] self: [5:7571519141913236000:0] peer: [2:7571519142720945372:0] socket: 631 qp: -1
2025-11-11T17:21:16.023013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519149082915355:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:16.263646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519146208203310:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:16.263682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:16.023081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:16.264812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519146625733205:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9213
2025-11-11T17:21:16.023545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519144787948033:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.221921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519146706238792:2080] [node 5] ICS10 traffic start
2025-11-11T17:21:16.023575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:16.023618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519144787948033:2107] [node 4] ICP22 created new session: [1:7571519149082915357:2107]
2025-11-11T17:21:16.226133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:16.271692Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28616) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:16.271854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519139785404579:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28616) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519148375339262:2119] held: no
2025-11-11T17:21:16.265132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519147015912697:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55
2025-11-11T17:21:16.247304Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519142411271462:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.023659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519149082915357:2107] [node 4] ICS09 handshake done sender: [1:7571519144787948037:2111] self: [1:7571519144787948036:0] peer: [4:7571519144080371958:0] socket: 622 qp: -1
2025-11-11T17:21:16.247354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519142411271462:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:16.305837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519147015912690:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:16.308623Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:16.309834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519138425978002:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519147015912690:2119] held: no
2025-11-11T17:21:16.293003Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:16.023690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519149082915357:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:16.296024Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519146625733199:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:16.023714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:16.262240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519142411271448:2080] [node 5] ICP35 event processing took too much time PendingConnection:278(64279296ns) {HandleHandshakeStatus:329(64273511ns) {DropIncomingHandshake:441(32025ns) DropOutgoingHandshake:456(467ns) SwitchToState:178(405ns) ProcessPendingSessionEvents:495(12030766ns) {ForwardSessionEventToSession:612(11958267ns) ForwardSessionEventToSession:612(14003ns) ForwardSessionEventToSession:612(11606ns) ForwardSessionEventToSession:612(7109ns) ForwardSessionEventToSession:612(3776ns) ForwardSessionEventToSession:612(3285ns) ForwardSessionEventToSession:612(6613ns) ForwardSessionEventToSession:612(3299ns) ForwardSessionEventToSession:612(3286ns) ForwardSessionEventToSession:612(6168ns)}}}
2025-11-11T17:21:16.311332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519142330765808:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519146625733199:2123] held: no
2025-11-11T17:21:16.312310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519142330765808:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17655226ns) {HandleHandshakeStatus:395(4739883ns) {LogHandshakeFail:472(3525390ns) {UpdateErrorStateLog:544(6760ns)} DropOutgoingHandshake:456(1928ns) ProcessPendingSessionEvents:495(453336ns) {DropSessionEvent:512(19348ns) DropSessionEvent:512(12068ns) DropSessionEvent:512(11643ns) DropSessionEvent:512(6490ns) DropSessionEvent:512(4950ns) DropSessionEvent:512(6564ns) DropSessionEvent:512(367214ns) DropSessionEvent:512(10044ns) DropSessionEvent:512(4928ns) DropSessionEvent:512(5427ns)}}}
2025-11-11T17:21:16.312395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519142330765808:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.026038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948054:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.098161Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519140492980673:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(159148875ns) {HandleHandshakeStatus:329(153819123ns) {DropIncomingHandshake:441(25811ns) DropOutgoingHandshake:456(452ns) SwitchToState:178(480ns) ProcessPendingSessionEvents:495(31020692ns) {ForwardSessionEventToSession:612(22787029ns) ForwardSessionEventToSession:612(8222ns) ForwardSessionEventToSession:612(4589ns) ForwardSessionEventToSession:612(1029951ns) ForwardSessionEventToSession:612(3486ns) ForwardSessionEventToSession:612(7153778ns) ForwardSessionEventToSession:612(6837ns) ForwardSessionEventToSession:612(2960ns) ForwardSessionEventToSession:612(5781ns) ForwardSessionEventToSession:612(2765ns)}}}
2025-11-11T17:21:16.187160Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948051:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.187218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519144787948051:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:16.233648Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519144787948047:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:16.233708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519144787948047:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:16.247383Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519144787948051:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:16.254162Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519144787948047:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:16.278429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519144787948031:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.278539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:16.278604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519144787948031:2109] [node 6] ICP22 created new session: [1:7571519149082915364:2109]
2025-11-11T17:21:16.278662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519149082915364:2109] [node 6] ICS09 handshake done sender: [1:7571519144787948051:2116] self: [1:7571519144787948048:0] peer: [6:7571519146625733204:0] socket: 606 qp: -1
2025-11-11T17:21:16.278703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519149082915364:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:16.278745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:16.279802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519140492980672:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.279866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:16.279921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519140492980672:2048] [node 2] ICP22 created new session: [1:7571519149082915366:2048]
2025-11-11T17:21:16.279974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519149082915366:2048] [node 2] ICS09 handshake done sender: [1:7571519144787948047:2115] self: [1:7571519144787948046:0] peer: [2:7571519147015912696:0] socket: 649 qp: -1
2025-11-11T17:21:16.280015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519149082915366:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:16.285751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:16.311431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519138425978002:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:16.314073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:16.315245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519142330765808:2048] [node 1] ICP22 created new session: [6:7571519146625733206:2048]
2025-11-11T17:21:16.315307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519146625733206:2048] [node 1] ICS09 handshake done sender: [6:7571519146625733199:2123] self: [6:7571519146625733204:0] peer: [1:7571519144787948048:0] socket: 605 qp: -1
2025-11-11T17:21:16.315353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519146625733206:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:16.315466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:16.311753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:16.311913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519138425978002:2048] [node 1] ICP22 created new session: [2:7571519147015912698:2048]
2025-11-11T17:21:16.311973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519147015912698:2048] [node 1] ICS09 handshake done sender: [2:7571519147015912690:2119] self: [2:7571519147015912696:0] peer: [1:7571519144787948046:0] socket: 652 qp: -1
2025-11-11T17:21:16.312020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519147015912698:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:16.314202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:16.269959Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519142411271462:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:16.270301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519142411271442:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:16.270410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:16.270464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519142411271442:2070] [node 4] ICP22 created new session: [3:7571519146706238795:2070]
2025-11-11T17:21:16.270514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519146706238795:2070] [node 4] ICS09 handshake done sender: [3:7571519142411271462:2110] self: [3:7571519142411271461:0] peer: [4:7571519148375339268:0] socket: 627 qp: -1
2025-11-11T17:21:16.270556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519146706238795:2070] [node 4] ICS10 traffic start
2025-11-11T17:21:16.270589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:16.316509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519148375339269:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23074
2025-11-11T17:21:16.317402Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519148375339262:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:16.317831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519139785404579:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:16.317979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:16.318032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519139785404579:2050] [node 3] ICP22 created new session: [4:7571519148375339270:2050]
2025-11-11T17:21:16.318090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519148375339270:2050] [node 3] ICS09 handshake done sender: [4:7571519148375339262:2119] self: [4:7571519148375339268:0] peer: [3:7571519142411271461:0] socket: 656 qp: -1
2025-11-11T17:21:16.318126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519148375339270:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:16.318174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:16.392496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:16.392749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519147015912698:2048] [node 1] ICS01 socket: 652 reason# EndOfStream
2025-11-11T17:21:16.392792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519138425978002:2048] [node 1] ICP30 unregister session Session# [2:7571519147015912698:2048] VirtualId# [2:7571519147015912696:0]
2025-11-11T17:21:16.392827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519147015912698:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.392911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:16.392955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:16.395235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:16.395477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519146706238787:2048] [node 1] ICS01 socket: 626 reason# EndOfStream
2025-11-11T17:21:16.395528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519138116304099:2048] [node 1] ICP30 unregister session Session# [3:7571519146706238787:2048] VirtualId# [3:7571519142411271482:0]
2025-11-11T17:21:16.395566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519146706238787:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.395662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:16.395718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:16.401798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:16.401817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:16.405923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519146625733206:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-11-11T17:21:16.405988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519142330765808:2048] [node 1] ICP30 unregister session Session# [6:7571519146625733206:2048] VirtualId# [6:7571519146625733204:0]
2025-11-11T17:21:16.406025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519146625733206:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.406138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:16.406045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519146208203302:2048] [node 1] ICS01 socket: 623 reason# EndOfStream
2025-11-11T17:21:16.405758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:16.406087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519141913235914:2048] [node 1] ICP30 unregister session Session# [5:7571519146208203302:2048] VirtualId# [5:7571519141913235999:0]
2025-11-11T17:21:16.406115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519146208203302:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.406195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:16.406247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:16.406196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:16.409872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519148375339261:2048] [node 1] ICS01 socket: 633 reason# EndOfStream
2025-11-11T17:21:16.409935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519139785404577:2048] [node 1] ICP30 unregister session Session# [4:7571519148375339261:2048] VirtualId# [4:7571519144080371958:0]
2025-11-11T17:21:16.409974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519148375339261:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.410094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:16.410140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:16.488881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:16.489035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519146706238783:2049] [node 2] ICS01 socket: 615 reason# EndOfStream
2025-11-11T17:21:16.489071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519138116304100:2049] [node 2] ICP30 unregister session Session# [3:7571519146706238783:2049] VirtualId# [3:7571519142411271478:0]
2025-11-11T17:21:16.489097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519146706238783:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.489154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:16.489184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:16.519314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:16.519577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519148375339266:2049] [node 2] ICS01 socket: 645 reason# EndOfStream
2025-11-11T17:21:16.519617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519139785404578:2049] [node 2] ICP30 unregister session Session# [4:7571519148375339266:2049] VirtualId# [4:7571519148375339260:0]
2025-11-11T17:21:16.519656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519148375339266:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.519726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:16.519789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:16.519896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:16.520055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519146208203310:2049] [node 2] ICS01 socket: 631 reason# EndOfStream
2025-11-11T17:21:16.520081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519141913235915:2049] [node 2] ICP30 unregister session Session# [5:7571519146208203310:2049] VirtualId# [5:7571519141913236000:0]
2025-11-11T17:21:16.520110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519146208203310:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.520163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:16.520196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:16.524216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:16.524496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519146625733202:2049] [node 2] ICS01 socket: 616 reason# EndOfStream
2025-11-11T17:21:16.524544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519142330765809:2049] [node 2] ICP30 unregister session Session# [6:7571519146625733202:2049] VirtualId# [6:7571519142330765897:0]
2025-11-11T17:21:16.524580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519146625733202:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.524692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:16.524743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:16.585797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:16.586023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519146625733200:2050] [node 3] ICS01 socket: 608 reason# EndOfStream
2025-11-11T17:21:16.586071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519142330765810:2050] [node 3] ICP30 unregister session Session# [6:7571519146625733200:2050] VirtualId# [6:7571519142330765896:0]
2025-11-11T17:21:16.586107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519146625733200:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.586206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:16.586248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:16.585770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:16.586027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519148375339270:2050] [node 3] ICS01 socket: 656 reason# EndOfStream
2025-11-11T17:21:16.586071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519139785404579:2050] [node 3] ICP30 unregister session Session# [4:7571519148375339270:2050] VirtualId# [4:7571519148375339268:0]
2025-11-11T17:21:16.586105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519148375339270:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.586205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:16.586247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:16.586326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:16.586554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519146208203308:2050] [node 3] ICS01 socket: 657 reason# EndOfStream
2025-11-11T17:21:16.586588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519141913235916:2050] [node 3] ICP30 unregister session Session# [5:7571519146208203308:2050] VirtualId# [5:7571519146208203306:0]
2025-11-11T17:21:16.586613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519146208203308:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.586670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:16.586704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:16.657682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:16.657916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519142330765898:2060] [node 4] ICS01 socket: 603 reason# EndOfStream
2025-11-11T17:21:16.657964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519142330765822:2060] [node 4] ICP30 unregister session Session# [6:7571519142330765898:2060] VirtualId# [6:7571519142330765887:0]
2025-11-11T17:21:16.657998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519142330765898:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.658080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:16.658121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:16.658593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:16.658778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519146208203300:2068] [node 4] ICS01 socket: 600 reason# EndOfStream
2025-11-11T17:21:16.658809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519141913235936:2068] [node 4] ICP30 unregister session Session# [5:7571519146208203300:2068] VirtualId# [5:7571519141913236001:0]
2025-11-11T17:21:16.658833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519146208203300:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.658889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:16.658929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:16.709784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:16.710005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519142330765894:2071] [node 5] ICS01 socket: 604 reason# EndOfStream
2025-11-11T17:21:16.710047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519142330765840:2071] [node 5] ICP30 unregister session Session# [6:7571519142330765894:2071] VirtualId# [6:7571519142330765882:0]
2025-11-11T17:21:16.710080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519142330765894:2071] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:16.710170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:16.710219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:16.856354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519148683086417:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:16.856421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519148683086418:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:16.876874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519148333200322:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:16.877110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519148333200321:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:16.906882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519148580538262:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:16.907236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519148580538261:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:16.945841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519148986263984:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:16.945889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519148986263985:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:16.945923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519148986263986:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:16.976653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519145174661085:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:16.976685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519145174661086:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:16.976706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519145174661087:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:17.004872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519151571434638:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:17.004922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519151571434640:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:17.004995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519151571434639:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:17.248026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519152978053775:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:17.248365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519152978053776:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:17.265564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519152628167679:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:17.362033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519153281231343:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:17.314077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519152978053778:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:17.380508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519152875505620:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:17.393038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519152628167681:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:17.401732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519152875505621:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:17.399288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519149469628444:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:17.398352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519151571434699:2100] [node 5] ICP01 ready to work
2025-11-11T17:21:17.409140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519152978053775:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(150705310ns) {Configure:104(150698222ns) {ProcessConfigured:135(11334757ns) {StartInitialHandshake:157(11331389ns) {DropHandshakes:469(2685ns) {DropIncomingHandshake:441(864ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(1031ns) PrepareNewSessionHandshake:476(3943ns) {SwitchToState:178(1760ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.417873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519152978053781:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1265
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.399403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519153281231347:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:17.393394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519152628167682:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:17.429994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519148683086417:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(181336058ns) {Configure:104(181331280ns) {ProcessConfigured:135(10017094ns) {StartInitialHandshake:157(10013263ns) {DropHandshakes:469(4224ns) {DropIncomingHandshake:441(797ns) DropOutgoingHandshake:456(756ns)} GenerateSessionVirtualId:988(1158ns) PrepareNewSessionHandshake:476(6224ns) {SwitchToState:178(3302ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.437036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519148333200321:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(178520418ns) {Configure:104(178515165ns) {ProcessConfigured:135(72602ns) {StartInitialHandshake:157(70126ns) {DropHandshakes:469(2760ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(1050ns) PrepareNewSessionHandshake:476(2445ns) {SwitchToState:178(1028ns)}}}}}
2025-11-11T17:21:17.437562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519152628167685:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-11T17:21:17.406224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519151571434700:2102] [node 4] ICP01 ready to work
2025-11-11T17:21:17.409294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519149469628445:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:17.418645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231348:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.408995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519152875505622:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.459264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519145174661086:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(144476545ns) {Configure:104(144471394ns) {ProcessConfigured:135(52969ns) {StartInitialHandshake:157(50711ns) {DropHandshakes:469(2057ns) {DropIncomingHandshake:441(413ns) DropOutgoingHandshake:456(467ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:476(1597ns) {SwitchToState:178(1038ns)}}}}}
2025-11-11T17:21:17.459523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628446:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
2025-11-11T17:21:17.458400Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519148333200322:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(160033920ns) {Configure:104(160025106ns) {ProcessConfigured:135(63043ns) {StartInitialHandshake:157(60004ns) {DropHandshakes:469(2501ns) {DropIncomingHandshake:441(835ns) DropOutgoingHandshake:456(488ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:476(2121ns) {SwitchToState:178(951ns)}}}}}
2025-11-11T17:21:17.451648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519151571434639:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(190805582ns) {Configure:104(190796564ns) {ProcessConfigured:135(15854505ns) {StartInitialHandshake:157(11160548ns) {DropHandshakes:469(3147ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(412ns)} GenerateSessionVirtualId:988(1787ns) PrepareNewSessionHandshake:476(4964ns) {SwitchToState:178(2555ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.469281Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519149469628446:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.482645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519152628167690:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-11T17:21:17.488956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519152628167679:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(223040969ns) {Configure:104(223036365ns) {ProcessConfigured:135(86394ns) {StartInitialHandshake:157(81933ns) {DropHandshakes:469(2636ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(421ns)} GenerateSessionVirtualId:988(1283ns) PrepareNewSessionHandshake:476(1892ns) {SwitchToState:178(834ns)}}}}}
2025-11-11T17:21:17.494382Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053781:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.478137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519148580538261:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(85345594ns) {Configure:104(85337406ns) {ProcessConfigured:135(65482ns) {StartInitialHandshake:157(63342ns) {DropHandshakes:469(3386ns) {DropIncomingHandshake:441(1052ns) DropOutgoingHandshake:456(479ns)} GenerateSessionVirtualId:988(1198ns) PrepareNewSessionHandshake:476(1553ns) {SwitchToState:178(677ns)}}}}}
2025-11-11T17:21:17.497364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519148683086418:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(232849244ns) {Configure:104(232845744ns) {ProcessConfigured:135(16757843ns) {StartInitialHandshake:157(16754818ns) {DropHandshakes:469(1330ns) {DropIncomingHandshake:441(330ns) DropOutgoingHandshake:456(298ns)} GenerateSessionVirtualId:988(722ns) PrepareNewSessionHandshake:476(5710ns) {SwitchToState:178(3142ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.509908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231349:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.518063Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167685:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.518394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519152978053776:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(241150051ns) {Configure:104(241139099ns) {ProcessConfigured:135(23560621ns) {StartInitialHandshake:157(23558537ns) {DropHandshakes:469(1266ns) {DropIncomingHandshake:441(317ns) DropOutgoingHandshake:456(277ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:476(3547ns) {SwitchToState:178(1537ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.522768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519149469628444:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(62938719ns) {Configure:104(62932553ns) {ProcessConfigured:135(18790580ns) {StartInitialHandshake:157(18788214ns) {DropHandshakes:469(2196ns) {DropIncomingHandshake:441(646ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:476(6673ns) {SwitchToState:178(3569ns)}}}}}
2025-11-11T17:21:17.526821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519148333200321:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.514867Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519153281231349:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.517418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519151571434640:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(180578983ns) {Configure:104(180574107ns) {ProcessConfigured:135(11928995ns) {StartInitialHandshake:157(11927103ns) {DropHandshakes:469(3303ns) {DropIncomingHandshake:441(616ns) DropOutgoingHandshake:456(455ns)} GenerateSessionVirtualId:988(1159ns) PrepareNewSessionHandshake:476(3897ns) {SwitchToState:178(2281ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.536775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519152978053786:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1265
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.549307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519148580538262:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(236004185ns) {Configure:104(235998570ns) {ProcessConfigured:135(22576986ns) {StartInitialHandshake:157(22573394ns) {DropHandshakes:469(1535ns) {DropIncomingHandshake:441(347ns) DropOutgoingHandshake:456(231ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:476(4368ns) {SwitchToState:178(1152ns)}}}}}
2025-11-11T17:21:17.553665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519151571434700:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(119262277ns) {Configure:104(119255409ns) {ProcessConfigured:135(68623ns) {StartInitialHandshake:157(66277ns) {DropHandshakes:469(2251ns) {DropIncomingHandshake:441(414ns) DropOutgoingHandshake:456(468ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:476(1685ns) {SwitchToState:178(763ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.537052Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519153281231345:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.536082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519149469628444:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.551457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519145174661085:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(261789805ns) {Configure:104(261783601ns) {ProcessConfigured:135(12829878ns) {StartInitialHandshake:157(9048567ns) {DropHandshakes:469(2896ns) {DropIncomingHandshake:441(319ns) DropOutgoingHandshake:456(339ns)} GenerateSessionVirtualId:988(9444ns) PrepareNewSessionHandshake:476(3234ns) {SwitchToState:178(1264ns)}}}}}
2025-11-11T17:21:17.569262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519145174661087:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(175013020ns) {Configure:104(175005946ns) {ProcessConfigured:135(7572949ns) {StartInitialHandshake:157(7570711ns) {DropHandshakes:469(579ns) {DropIncomingHandshake:441(193ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(2521ns) {SwitchToState:178(847ns)}}}}}
2025-11-11T17:21:17.568112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519152875505620:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(161043917ns) {Configure:104(161036896ns) {ProcessConfigured:135(12885514ns) {StartInitialHandshake:157(7674888ns) {DropHandshakes:469(1599ns) {DropIncomingHandshake:441(1161ns) DropOutgoingHandshake:456(177ns)} GenerateSessionVirtualId:988(927ns) PrepareNewSessionHandshake:476(2213ns) {SwitchToState:178(1261ns)}}}}}
2025-11-11T17:21:17.578272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628459:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
2025-11-11T17:21:17.580687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519152875505627:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21345
2025-11-11T17:21:17.577904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434713:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.548091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519152628167689:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.583070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519152875505621:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(174863350ns) {Configure:104(174854124ns) {ProcessConfigured:135(8868680ns) {StartInitialHandshake:157(8865179ns) {DropHandshakes:469(4575ns) {DropIncomingHandshake:441(1238ns) DropOutgoingHandshake:456(1081ns)} GenerateSessionVirtualId:988(1006ns) PrepareNewSessionHandshake:476(1975ns) {SwitchToState:178(523ns)}}}}}
2025-11-11T17:21:17.537081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519153281231345:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.553508Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519152628167682:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(121596281ns) {Configure:104(121593478ns) {ProcessConfigured:135(12537843ns) {StartInitialHandshake:157(12535727ns) {DropHandshakes:469(2095ns) {DropIncomingHandshake:441(410ns) DropOutgoingHandshake:456(335ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:476(3221ns) {SwitchToState:178(987ns)}}}}}
2025-11-11T17:21:17.540709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053784:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.583229Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505627:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.540749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519152978053784:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.561948Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167693:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.589057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519151571434699:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(169050346ns) {Configure:104(169045799ns) {ProcessConfigured:135(5637347ns) {StartInitialHandshake:157(5635407ns) {DropHandshakes:469(1702ns) {DropIncomingHandshake:441(346ns) DropOutgoingHandshake:456(314ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:476(3085ns) {SwitchToState:178(1555ns)}}}}}
2025-11-11T17:21:17.550663Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519153281231348:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.584262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519148580538262:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.561991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519152628167693:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.551133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231352:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.553957Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519152978053784:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:17.594453Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519151571434713:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.562926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519152978053796:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1265
2025-11-11T17:21:17.573898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519152978053795:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1265
2025-11-11T17:21:17.581142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519152978053778:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(171295631ns) {Configure:104(171285780ns) {ProcessConfigured:135(23569492ns) {StartInitialHandshake:157(23565633ns) {DropHandshakes:469(2453ns) {DropIncomingHandshake:441(867ns) DropOutgoingHandshake:456(408ns)} GenerateSessionVirtualId:988(905ns) PrepareNewSessionHandshake:476(2830ns) {SwitchToState:178(1928ns)}}}}}
2025-11-11T17:21:17.582069Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053786:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.585884Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053780:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.585921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519152978053780:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.586245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519148683086417:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.586280Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053795:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.589194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:17.592014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519148683086417:2048] [node 2] ICP22 created new session: [1:7571519152978053800:2048]
2025-11-11T17:21:17.597195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519152978053800:2048] [node 2] ICS09 handshake done sender: [1:7571519152978053784:2112] self: [1:7571519152978053783:0] peer: [2:7571519152628167697:0] socket: 606 qp: -1
2025-11-11T17:21:17.598583Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053796:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.605783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519152978053800:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:17.605841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.585872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519152875505628:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21345
2025-11-11T17:21:17.595802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434717:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.567967Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167690:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.627606Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519152978053780:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.615929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519149469628445:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(154756025ns) {Configure:104(154748917ns) {ProcessConfigured:135(11092548ns) {StartInitialHandshake:157(8805592ns) {DropHandshakes:469(3011ns) {DropIncomingHandshake:441(356ns) DropOutgoingHandshake:456(444ns)} GenerateSessionVirtualId:988(1380ns) PrepareNewSessionHandshake:476(4681ns) {SwitchToState:178(2100ns)}}}}}
2025-11-11T17:21:17.594834Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505628:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.554505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519153281231347:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(142505722ns) {Configure:104(142495629ns) {ProcessConfigured:135(50190ns) {StartInitialHandshake:157(47931ns) {DropHandshakes:469(3020ns) {DropIncomingHandshake:441(651ns) DropOutgoingHandshake:456(399ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:476(1901ns) {SwitchToState:178(919ns)}}}}}
2025-11-11T17:21:17.629936Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053789:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.597288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519151571434700:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.573313Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519152628167693:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:17.555134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519148986263985:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(162090323ns) {Configure:104(162079218ns) {ProcessConfigured:135(7703887ns) {StartInitialHandshake:157(7701800ns) {DropHandshakes:469(2851ns) {DropIncomingHandshake:441(522ns) DropOutgoingHandshake:456(640ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:476(2179ns) {SwitchToState:178(655ns)}}}}}
2025-11-11T17:21:17.632318Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519149469628446:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:17.613481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519152875505622:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(152927867ns) {Configure:104(152920633ns) {ProcessConfigured:135(6217644ns) {StartInitialHandshake:157(6214341ns) {DropHandshakes:469(3441ns) {DropIncomingHandshake:441(822ns) DropOutgoingHandshake:456(467ns)} GenerateSessionVirtualId:988(1164ns) PrepareNewSessionHandshake:476(3099ns) {SwitchToState:178(1584ns)}}}}}
2025-11-11T17:21:17.577798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519152628167681:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(157981200ns) {Configure:104(157973133ns) {ProcessConfigured:135(59361ns) {StartInitialHandshake:157(56061ns) {DropHandshakes:469(2205ns) {DropIncomingHandshake:441(501ns) DropOutgoingHandshake:456(483ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:476(1502ns) {SwitchToState:178(635ns)}}}}}
2025-11-11T17:21:17.642462Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519151571434717:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.629968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519152978053789:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.555282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519148986263985:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.634262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628462:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.614114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519148580538261:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.581413Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167689:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.555797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519148986263984:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(162347575ns) {Configure:104(162342025ns) {ProcessConfigured:135(7368800ns) {StartInitialHandshake:157(7364195ns) {DropHandshakes:469(2659ns) {DropIncomingHandshake:441(570ns) DropOutgoingHandshake:456(642ns)} GenerateSessionVirtualId:988(1058ns) PrepareNewSessionHandshake:476(5399ns) {SwitchToState:178(2654ns)}}}}}
2025-11-11T17:21:17.650209Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519152978053789:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:17.657209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519151571434639:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.664268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519152875505642:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21345
2025-11-11T17:21:17.644958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628463:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.556053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519148986263984:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.655947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519152978053775:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.582600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519152628167698:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-11T17:21:17.675139Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29325) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.582721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519152628167679:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.655456Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519149469628463:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.661331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519148683086418:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.557757Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519153281231345:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:17.664589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519152875505641:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21345
2025-11-11T17:21:17.679653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519151571434700:2102] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29325) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519151571434713:2114] held: no
2025-11-11T17:21:17.583827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:17.586392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519152628167679:2107] [node 4] ICP22 created new session: [2:7571519152628167703:2107]
2025-11-11T17:21:17.590462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519152628167703:2107] [node 4] ICS09 handshake done sender: [2:7571519152628167693:2115] self: [2:7571519152628167692:0] peer: [4:7571519153281231358:0] socket: 615 qp: -1
2025-11-11T17:21:17.591560Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519152628167685:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:17.595076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519152628167703:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:17.598366Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167688:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.598405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519152628167688:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.599288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:17.611512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519148333200321:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.611575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:17.611637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519148333200321:2048] [node 1] ICP22 created new session: [2:7571519152628167708:2048]
2025-11-11T17:21:17.611686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519152628167708:2048] [node 1] ICS09 handshake done sender: [2:7571519152628167685:2111] self: [2:7571519152628167697:0] peer: [1:7571519152978053783:0] socket: 607 qp: -1
2025-11-11T17:21:17.611731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519152628167708:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:17.611803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:17.612577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519152628167707:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-11T17:21:17.612916Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.615894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519148333200321:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:17.630805Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519152628167688:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:17.631006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519148333200322:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.631075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:17.631144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519148333200322:2049] [node 3] ICP22 created new session: [2:7571519152628167712:2049]
2025-11-11T17:21:17.631195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519152628167712:2049] [node 3] ICS09 handshake done sender: [2:7571519152628167688:2112] self: [2:7571519152628167687:0] peer: [3:7571519152875505636:0] socket: 614 qp: -1
2025-11-11T17:21:17.631232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519152628167712:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:17.631285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:17.645118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519152628167679:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(62141692ns) {HandleHandshakeStatus:329(62138884ns) {DropIncomingHandshake:441(191ns) DropOutgoingHandshake:456(450ns) SwitchToState:178(1481ns) ProcessPendingSessionEvents:495(12434303ns) {ForwardSessionEventToSession:612(12366507ns) ForwardSessionEventToSession:612(14919ns) ForwardSessionEventToSession:612(4818ns) ForwardSessionEventToSession:612(5754ns) ForwardSessionEventToSession:612(3092ns) ForwardSessionEventToSession:612(5194ns) ForwardSessionEventToSession:612(6037ns) ForwardSessionEventToSession:612(3284ns) ForwardSessionEventToSession:612(3112ns) ForwardSessionEventToSession:612(5126ns)}}}
2025-11-11T17:21:17.651013Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167707:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.657008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519148333200322:2049] [node 3] ICP18 handshake (actor [2:7571519152628167707:2119]) is too old
2025-11-11T17:21:17.658826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167696:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.658861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519152628167696:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.672598Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519152628167700:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.672667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519152628167700:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.697796Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519152628167700:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:17.697817Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519152628167696:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:17.733791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519152628167681:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.733868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:17.734932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519152628167682:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.738189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:17.655668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519145174661086:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.661381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:17.661439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519148683086418:2049] [node 3] ICP22 created new session: [1:7571519152978053807:2049]
2025-11-11T17:21:17.661503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519152978053807:2049] [node 3] ICS09 handshake done sender: [1:7571519152978053789:2114] self: [1:7571519152978053787:0] peer: [3:7571519152875505639:0] socket: 617 qp: -1
2025-11-11T17:21:17.661537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519152978053807:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:17.661598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:17.672674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:17.681857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519152978053775:2107] [node 4] ICP22 created new session: [1:7571519152978053809:2107]
2025-11-11T17:21:17.709695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519152978053809:2107] [node 4] ICS09 handshake done sender: [1:7571519152978053780:2110] self: [1:7571519152978053779:0] peer: [4:7571519153281231360:0] socket: 604 qp: -1
2025-11-11T17:21:17.719327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519152978053809:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:17.722728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519148683086417:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(78281360ns) {HandleHandshakeStatus:329(78280032ns) {DropIncomingHandshake:441(34584ns) DropOutgoingHandshake:456(551ns) SwitchToState:178(354ns) ProcessPendingSessionEvents:495(11258566ns) {ForwardSessionEventToSession:612(5287402ns) ForwardSessionEventToSession:612(6641ns) ForwardSessionEventToSession:612(1722760ns) ForwardSessionEventToSession:612(901092ns) ForwardSessionEventToSession:612(6204ns) ForwardSessionEventToSession:612(1134747ns) ForwardSessionEventToSession:612(4081ns) ForwardSessionEventToSession:612(2170594ns) ForwardSessionEventToSession:612(3871ns) ForwardSessionEventToSession:612(2540ns)}}}
2025-11-11T17:21:17.722777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:17.726471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519152978053801:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1265
2025-11-11T17:21:17.734678Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:17.738024Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053790:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.738073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519152978053790:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.738350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519148683086418:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(20829173ns) {HandleHandshakeStatus:329(20826763ns) {DropIncomingHandshake:441(361ns) DropOutgoingHandshake:456(778ns) SwitchToState:178(363ns) ProcessPendingSessionEvents:495(20416910ns) {ForwardSessionEventToSession:612(54907ns) ForwardSessionEventToSession:612(7280ns) ForwardSessionEventToSession:612(9736ns) ForwardSessionEventToSession:612(686958ns) ForwardSessionEventToSession:612(12189540ns) ForwardSessionEventToSession:612(4394ns) ForwardSessionEventToSession:612(4265843ns) ForwardSessionEventToSession:612(16681ns) ForwardSessionEventToSession:612(3659ns) ForwardSessionEventToSession:612(3156579ns)}}}
2025-11-11T17:21:17.566334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519148986263986:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(173002796ns) {Configure:104(172996279ns) {ProcessConfigured:135(8944487ns) {StartInitialHandshake:157(8941078ns) {DropHandshakes:469(1867ns) {DropIncomingHandshake:441(396ns) DropOutgoingHandshake:456(366ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:476(3661ns) {SwitchToState:178(1305ns)}}}}}
2025-11-11T17:21:17.743382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519152628167681:2109] [node 6] ICP22 created new session: [2:7571519152628167716:2109]
2025-11-11T17:21:17.681938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434719:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.655907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519149469628444:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.665094Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519152875505628:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:17.762885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519152978053775:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(106728761ns) {HandleHandshakeStatus:329(106726483ns) {DropIncomingHandshake:441(12959ns) DropOutgoingHandshake:456(414ns) SwitchToState:178(442ns) ProcessPendingSessionEvents:495(19273434ns) {ForwardSessionEventToSession:612(14501486ns) ForwardSessionEventToSession:612(15335ns) ForwardSessionEventToSession:612(4019ns) ForwardSessionEventToSession:612(8943ns) ForwardSessionEventToSession:612(2988ns) ForwardSessionEventToSession:612(3352ns) ForwardSessionEventToSession:612(5064ns) ForwardSessionEventToSession:612(3016ns) ForwardSessionEventToSession:612(9790ns) ForwardSessionEventToSession:612(8433ns)}}}
2025-11-11T17:21:17.665348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519148580538261:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.603295Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519153281231355:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.726371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434722:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.753929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519152628167682:2108] [node 5] ICP22 created new session: [2:7571519152628167717:2108]
2025-11-11T17:21:17.655978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.763542Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519151571434713:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.754066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519152628167716:2109] [node 6] ICS09 handshake done sender: [2:7571519152628167700:2118] self: [2:7571519152628167699:0] peer: [6:7571519151571434720:0] socket: 639 qp: -1
2025-11-11T17:21:17.765123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519151571434722:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.765764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519151571434700:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.765826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:17.765883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519151571434700:2102] [node 4] ICP22 created new session: [6:7571519151571434724:2102]
2025-11-11T17:21:17.765951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519151571434724:2102] [node 4] ICS09 handshake done sender: [6:7571519151571434713:2114] self: [6:7571519151571434718:0] peer: [4:7571519153281231354:0] socket: 629 qp: -1
2025-11-11T17:21:17.765985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519151571434724:2102] [node 4] ICS10 traffic start
2025-11-11T17:21:17.766022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:17.766453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434723:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.771406Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.785903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519151571434639:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519151571434717:2116] held: no
2025-11-11T17:21:17.786561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519151571434640:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.788546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434727:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.801984Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519151571434723:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.802182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519151571434638:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.657607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519149469628444:2108] [node 4] ICP22 created new session: [5:7571519149469628465:2108]
2025-11-11T17:21:17.603327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519153281231355:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.665409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:17.789263Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519152978053790:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:17.658818Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519149469628462:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.661215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519149469628465:2108] [node 4] ICS09 handshake done sender: [5:7571519149469628446:2110] self: [5:7571519149469628452:0] peer: [4:7571519153281231344:0] socket: 602 qp: -1
2025-11-11T17:21:17.661287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519149469628465:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:17.661362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:17.673217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519145174661085:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.676609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628469:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
2025-11-11T17:21:17.701299Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519149469628463:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:17.727996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519145174661086:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.728707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628470:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
2025-11-11T17:21:17.759861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:17.764311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519145174661086:2049] [node 2] ICP22 created new session: [5:7571519149469628471:2049]
2025-11-11T17:21:17.772676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519149469628471:2049] [node 2] ICS09 handshake done sender: [5:7571519149469628463:2118] self: [5:7571519149469628464:0] peer: [2:7571519152628167695:0] socket: 648 qp: -1
2025-11-11T17:21:17.788910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519149469628471:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:17.804285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628473:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
2025-11-11T17:21:17.812348Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519149469628462:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:17.812837Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519149469628470:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.817791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:17.822334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519145174661085:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.822437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:17.822492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519145174661085:2048] [node 1] ICP22 created new session: [5:7571519149469628475:2048]
2025-11-11T17:21:17.822539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519149469628475:2048] [node 1] ICS09 handshake done sender: [5:7571519149469628462:2117] self: [5:7571519149469628466:0] peer: [1:7571519152978053788:0] socket: 644 qp: -1
2025-11-11T17:21:17.822583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519149469628475:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:17.822625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:17.665460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519148580538261:2048] [node 1] ICP22 created new session: [3:7571519152875505643:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.604212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519153281231343:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.813179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519152978053776:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.825637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519152978053794:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.825708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519152978053794:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.833553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519148683086418:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:17.759407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519152628167716:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:17.829822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519145174661087:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.821776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434721:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.763909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:17.849398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628477:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.604303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:17.665505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519152875505643:2048] [node 1] ICS09 handshake done sender: [3:7571519152875505628:2113] self: [3:7571519152875505639:0] peer: [1:7571519152978053787:0] socket: 619 qp: -1
2025-11-11T17:21:17.838134Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519151571434717:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:17.764038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519152628167717:2108] [node 5] ICS09 handshake done sender: [2:7571519152628167696:2116] self: [2:7571519152628167695:0] peer: [5:7571519149469628464:0] socket: 608 qp: -1
2025-11-11T17:21:17.665541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519152875505643:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:17.850691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:17.862565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519152978053776:2108] [node 5] ICP22 created new session: [1:7571519152978053813:2108]
2025-11-11T17:21:17.868229Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519152978053794:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:17.875515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519152978053813:2108] [node 5] ICS09 handshake done sender: [1:7571519152978053790:2115] self: [1:7571519152978053788:0] peer: [5:7571519149469628466:0] socket: 642 qp: -1
2025-11-11T17:21:17.866848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519149469628477:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.604368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519153281231343:2108] [node 5] ICP22 created new session: [4:7571519153281231367:2108]
2025-11-11T17:21:17.665574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:17.672845Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.682031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519148580538262:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519152875505627:2112] held: no
2025-11-11T17:21:17.706293Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505641:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.748175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519148580538262:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(48357421ns) {HandleHandshakeStatus:395(48353058ns) {LogHandshakeFail:472(16250ns) {UpdateErrorStateLog:544(5213ns)} DropOutgoingHandshake:456(2460ns) ProcessPendingSessionEvents:495(10741310ns) {DropSessionEvent:512(28312ns) DropSessionEvent:512(7165214ns) DropSessionEvent:512(6940ns) DropSessionEvent:512(5551ns) DropSessionEvent:512(16435ns) DropSessionEvent:512(6233ns) DropSessionEvent:512(6867ns) DropSessionEvent:512(3460455ns) DropSessionEvent:512(5076ns) DropSessionEvent:512(5883ns)}}}
2025-11-11T17:21:17.749248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519152875505640:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21345
2025-11-11T17:21:17.765292Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519152875505627:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:17.767138Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505631:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.767187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519152875505631:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.772728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519152875505648:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21345
2025-11-11T17:21:17.781281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519148580538262:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.781335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:17.781390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519148580538262:2049] [node 2] ICP22 created new session: [3:7571519152875505649:2049]
2025-11-11T17:21:17.781439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519152875505649:2049] [node 2] ICS09 handshake done sender: [3:7571519152875505627:2112] self: [3:7571519152875505636:0] peer: [2:7571519152628167687:0] socket: 618 qp: -1
2025-11-11T17:21:17.781477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519152875505649:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:17.781520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:17.794665Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519152875505631:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:17.804399Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505637:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.804424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519152875505637:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.805362Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505648:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.819133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519152875505620:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.825840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:17.828112Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519152875505637:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:17.828152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519152875505652:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21345
2025-11-11T17:21:17.828193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519152875505620:2107] [node 4] ICP22 created new session: [3:7571519152875505655:2107]
2025-11-11T17:21:17.838237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519152875505655:2107] [node 4] ICS09 handshake done sender: [3:7571519152875505631:2114] self: [3:7571519152875505629:0] peer: [4:7571519153281231378:0] socket: 637 qp: -1
2025-11-11T17:21:17.848489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519152875505622:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.848545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:17.848602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519152875505622:2109] [node 6] ICP22 created new session: [3:7571519152875505656:2109]
2025-11-11T17:21:17.848655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519152875505656:2109] [node 6] ICS09 handshake done sender: [3:7571519152875505637:2116] self: [3:7571519152875505634:0] peer: [6:7571519151571434728:0] socket: 609 qp: -1
2025-11-11T17:21:17.848693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519152875505656:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:17.848732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:17.849526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519152875505655:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:17.857797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:17.865690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505635:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.865747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519152875505635:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.867141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519152875505652:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.874861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519152875505622:2109] [node 6] ICP18 handshake (actor [3:7571519152875505652:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.883569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519152978053813:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:17.883672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:17.838215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519151571434727:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.765238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519152628167717:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.839854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519151571434699:2100] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.881640Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519149469628461:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.896158Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519152875505635:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:17.801871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:17.899872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519152978053778:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.906243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519152875505621:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.881697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519149469628461:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:17.901267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:17.810416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519152628167681:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(76379183ns) {HandleHandshakeStatus:329(76374932ns) {DropIncomingHandshake:441(19113ns) DropOutgoingHandshake:456(727ns) SwitchToState:178(1123ns) ProcessPendingSessionEvents:495(33190336ns) {ForwardSessionEventToSession:612(12761568ns) ForwardSessionEventToSession:612(8033698ns) ForwardSessionEventToSession:612(3973ns) ForwardSessionEventToSession:612(12334607ns) ForwardSessionEventToSession:612(5744ns) ForwardSessionEventToSession:612(3939ns) ForwardSessionEventToSession:612(6259ns) ForwardSessionEventToSession:612(3372ns) ForwardSessionEventToSession:612(5785ns) ForwardSessionEventToSession:612(5481ns)}}}
2025-11-11T17:21:17.857896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519151571434639:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:17.899780Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519149469628461:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:17.906347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:17.823290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519152628167682:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(88133574ns) {HandleHandshakeStatus:329(88130483ns) {DropIncomingHandshake:441(10805ns) DropOutgoingHandshake:456(311ns) SwitchToState:178(471ns) ProcessPendingSessionEvents:495(10178476ns) {ForwardSessionEventToSession:612(10121710ns) ForwardSessionEventToSession:612(9889ns) ForwardSessionEventToSession:612(2970ns) ForwardSessionEventToSession:612(2499ns) ForwardSessionEventToSession:612(4842ns) ForwardSessionEventToSession:612(3340ns) ForwardSessionEventToSession:612(8098ns) ForwardSessionEventToSession:612(5371ns) ForwardSessionEventToSession:612(2910ns) ForwardSessionEventToSession:612(5127ns)}}}
2025-11-11T17:21:17.901316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519152978053778:2109] [node 6] ICP22 created new session: [1:7571519152978053816:2109]
2025-11-11T17:21:17.866459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434733:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.604417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519153281231367:2108] [node 5] ICS09 handshake done sender: [4:7571519153281231345:2110] self: [4:7571519153281231344:0] peer: [5:7571519149469628452:0] socket: 600 qp: -1
2025-11-11T17:21:17.900851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519149469628481:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:614
2025-11-11T17:21:17.909433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519152875505621:2108] [node 5] ICP22 created new session: [3:7571519152875505660:2108]
2025-11-11T17:21:17.604462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519153281231367:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:17.907860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519152978053816:2109] [node 6] ICS09 handshake done sender: [1:7571519152978053794:2116] self: [1:7571519152978053792:0] peer: [6:7571519151571434729:0] socket: 659 qp: -1
2025-11-11T17:21:17.885698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:17.915657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519152875505660:2108] [node 5] ICS09 handshake done sender: [3:7571519152875505635:2115] self: [3:7571519152875505632:0] peer: [5:7571519149469628478:0] socket: 657 qp: -1
2025-11-11T17:21:17.907902Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519149469628470:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:17.909671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519152978053816:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:17.889491Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519151571434723:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:17.915712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519152875505660:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:17.604524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:17.893735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519151571434639:2049] [node 2] ICP22 created new session: [6:7571519151571434734:2049]
2025-11-11T17:21:17.893800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519151571434734:2049] [node 2] ICS09 handshake done sender: [6:7571519151571434717:2116] self: [6:7571519151571434720:0] peer: [2:7571519152628167699:0] socket: 640 qp: -1
2025-11-11T17:21:17.893840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519151571434734:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:17.893898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:17.894139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519151571434639:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(36201164ns) {HandleHandshakeStatus:329(36196832ns) {DropIncomingHandshake:441(1445ns) DropOutgoingHandshake:456(2007ns) SwitchToState:178(3692ns) ProcessPendingSessionEvents:495(1917ns)}}
2025-11-11T17:21:17.897964Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.898207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519151571434638:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.900104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519151571434640:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519151571434722:2119] held: no
2025-11-11T17:21:17.901013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:17.903224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434730:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.905994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519151571434638:2048] [node 1] ICP22 created new session: [6:7571519151571434736:2048]
2025-11-11T17:21:17.909428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519151571434736:2048] [node 1] ICS09 handshake done sender: [6:7571519151571434723:2120] self: [6:7571519151571434729:0] peer: [1:7571519152978053792:0] socket: 611 qp: -1
2025-11-11T17:21:17.909476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519151571434736:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:17.915496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:17.915851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:17.916712Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.917085Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519151571434722:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:17.920697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519152978053778:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(29217889ns) {HandleHandshakeStatus:329(23029343ns) {DropIncomingHandshake:441(29189ns) DropOutgoingHandshake:456(667ns) SwitchToState:178(2038ns) ProcessPendingSessionEvents:495(1035934ns) {ForwardSessionEventToSession:612(983093ns) ForwardSessionEventToSession:612(10206ns) ForwardSessionEventToSession:612(5134ns) ForwardSessionEventToSession:612(3105ns) ForwardSessionEventToSession:612(1979ns) ForwardSessionEventToSession:612(5829ns) ForwardSessionEventToSession:612(3150ns) ForwardSessionEventToSession:612(2744ns) ForwardSessionEventToSession:612(7166ns) ForwardSessionEventToSession:612(5233ns)}}}
2025-11-11T17:21:17.918144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519151571434699:2100] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519151571434727:2121] held: no
2025-11-11T17:21:17.921819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519152978053776:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(103804109ns) {HandleHandshakeStatus:329(103800593ns) {DropIncomingHandshake:441(5819785ns) DropOutgoingHandshake:456(1670ns) SwitchToState:178(655ns) ProcessPendingSessionEvents:495(15858021ns) {ForwardSessionEventToSession:612(8258018ns) ForwardSessionEventToSession:612(8274ns) ForwardSessionEventToSession:612(3625ns) ForwardSessionEventToSession:612(4084835ns) ForwardSessionEventToSession:612(6394ns) ForwardSessionEventToSession:612(1775052ns) ForwardSessionEventToSession:612(967476ns) ForwardSessionEventToSession:612(3198ns) ForwardSessionEventToSession:612(718028ns) ForwardSessionEventToSession:612(3391ns)}}}
2025-11-11T17:21:17.918364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519151571434737:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-11T17:21:17.920285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519151571434640:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.921558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:17.921696Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519151571434727:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:17.922383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519151571434640:2050] [node 3] ICP22 created new session: [6:7571519151571434739:2050]
2025-11-11T17:21:17.922432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519151571434699:2100] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.922480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:17.922512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519151571434699:2100] [node 5] ICP22 created new session: [6:7571519151571434740:2100]
2025-11-11T17:21:17.922572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519151571434740:2100] [node 5] ICS09 handshake done sender: [6:7571519151571434727:2121] self: [6:7571519151571434732:0] peer: [5:7571519149469628460:0] socket: 660 qp: -1
2025-11-11T17:21:17.922605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519151571434740:2100] [node 5] ICS10 traffic start
2025-11-11T17:21:17.922644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:17.924250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519151571434739:2050] [node 3] ICS09 handshake done sender: [6:7571519151571434722:2119] self: [6:7571519151571434728:0] peer: [3:7571519152875505634:0] socket: 656 qp: -1
2025-11-11T17:21:17.924295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519151571434739:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:17.924339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:17.918426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:17.919884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519152875505620:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(84086947ns) {HandleHandshakeStatus:329(84084128ns) {DropIncomingHandshake:441(15159ns) DropOutgoingHandshake:456(307ns) SwitchToState:178(354ns) ProcessPendingSessionEvents:495(22877613ns) {ForwardSessionEventToSession:612(9301361ns) ForwardSessionEventToSession:612(10134ns) ForwardSessionEventToSession:612(3055ns) ForwardSessionEventToSession:612(10071340ns) ForwardSessionEventToSession:612(17799ns) ForwardSessionEventToSession:612(1285761ns) ForwardSessionEventToSession:612(4270ns) ForwardSessionEventToSession:612(3630ns) ForwardSessionEventToSession:612(2149293ns) ForwardSessionEventToSession:612(3699ns)}}}
2025-11-11T17:21:17.908111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519149469628445:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.913487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:17.915064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519145174661086:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(173434605ns) {HandleHandshakeStatus:329(173429784ns) {DropIncomingHandshake:441(2408ns) DropOutgoingHandshake:456(25413ns) SwitchToState:178(745ns) ProcessPendingSessionEvents:495(54479825ns) {ForwardSessionEventToSession:612(25155356ns) ForwardSessionEventToSession:612(7184593ns) ForwardSessionEventToSession:612(10528ns) ForwardSessionEventToSession:612(3882ns) ForwardSessionEventToSession:612(7454557ns) ForwardSessionEventToSession:612(5488ns) ForwardSessionEventToSession:612(5113345ns) ForwardSessionEventToSession:612(4425332ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(5092067ns)}}}
2025-11-11T17:21:17.915117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519145174661087:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.915152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:17.915791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519149469628445:2109] [node 6] ICP22 created new session: [5:7571519149469628482:2109]
2025-11-11T17:21:17.916354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519145174661087:2050] [node 3] ICP22 created new session: [5:7571519149469628483:2050]
2025-11-11T17:21:17.916365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519149469628482:2109] [node 6] ICS09 handshake done sender: [5:7571519149469628461:2116] self: [5:7571519149469628460:0] peer: [6:7571519151571434732:0] socket: 613 qp: -1
2025-11-11T17:21:17.917182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519149469628482:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:17.918097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:17.919261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519149469628483:2050] [node 3] ICS09 handshake done sender: [5:7571519149469628470:2120] self: [5:7571519149469628478:0] peer: [3:7571519152875505632:0] socket: 658 qp: -1
2025-11-11T17:21:17.920371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519149469628483:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:17.920421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:17.927156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519151571434638:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28705414ns) {HandleHandshakeStatus:329(28699787ns) {DropIncomingHandshake:441(837ns) DropOutgoingHandshake:456(6955ns) SwitchToState:178(392ns) ProcessPendingSessionEvents:495(8339478ns) {ForwardSessionEventToSession:612(3003838ns) ForwardSessionEventToSession:612(3713ns) ForwardSessionEventToSession:612(260170ns) ForwardSessionEventToSession:612(3337ns) ForwardSessionEventToSession:612(1449850ns) ForwardSessionEventToSession:612(3236ns) ForwardSessionEventToSession:612(408963ns) ForwardSessionEventToSession:612(3180011ns) ForwardSessionEventToSession:612(4423ns) ForwardSessionEventToSession:612(3919ns)}}}
2025-11-11T17:21:17.927200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519152875505621:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(20745692ns) {HandleHandshakeStatus:329(20741761ns) {DropIncomingHandshake:441(24674ns) DropOutgoingHandshake:456(372ns) SwitchToState:178(453ns) ProcessPendingSessionEvents:495(6616211ns) {ForwardSessionEventToSession:612(6557738ns) ForwardSessionEventToSession:612(10377ns) ForwardSessionEventToSession:612(3122ns) ForwardSessionEventToSession:612(2959ns) ForwardSessionEventToSession:612(10220ns) ForwardSessionEventToSession:612(3002ns) ForwardSessionEventToSession:612(6054ns) ForwardSessionEventToSession:612(2460ns) ForwardSessionEventToSession:612(5292ns) ForwardSessionEventToSession:612(4580ns)}}}
2025-11-11T17:21:17.605242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231364:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.613822Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519153281231364:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.626555Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519153281231352:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.629828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519153281231355:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:17.638020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.642029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519153281231343:2108] [node 5] ICP18 handshake (actor [4:7571519153281231352:2113]) is too old
2025-11-11T17:21:17.649941Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519153281231349:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:17.655955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519153281231347:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:17.656474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519148986263985:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.656540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:17.656600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519148986263985:2049] [node 2] ICP22 created new session: [4:7571519153281231373:2049]
2025-11-11T17:21:17.657041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519153281231373:2049] [node 2] ICS09 handshake done sender: [4:7571519153281231349:2112] self: [4:7571519153281231358:0] peer: [2:7571519152628167692:0] socket: 616 qp: -1
2025-11-11T17:21:17.657860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519153281231373:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:17.659025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:17.663346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:17.675079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519153281231347:2109] [node 6] ICP22 created new session: [4:7571519153281231376:2109]
2025-11-11T17:21:17.681872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.718964Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.719494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519153281231376:2109] [node 6] ICS09 handshake done sender: [4:7571519153281231355:2114] self: [4:7571519153281231354:0] peer: [6:7571519151571434718:0] socket: 628 qp: -1
2025-11-11T17:21:17.719548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519153281231376:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:17.722008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519153281231366:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:17.726529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519148986263984:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519153281231348:2111] held: no
2025-11-11T17:21:17.734852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:17.737859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231371:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.746480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519148986263986:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:17.776689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519148986263984:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18817738ns) {HandleHandshakeStatus:395(18814841ns) {LogHandshakeFail:472(12727ns) {UpdateErrorStateLog:544(4020ns)} DropOutgoingHandshake:456(3088ns) ProcessPendingSessionEvents:495(11400861ns) {DropSessionEvent:512(12672ns) DropSessionEvent:512(20576ns) DropSessionEvent:512(6141ns) DropSessionEvent:512(13851ns) DropSessionEvent:512(5282ns) DropSessionEvent:512(4433ns) DropSessionEvent:512(5394ns) DropSessionEvent:512(929450ns) DropSessionEvent:512(8450ns) DropSessionEvent:512(25222ns)}}}
2025-11-11T17:21:17.777894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519153281231347:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(121712891ns) {HandleHandshakeStatus:329(121710335ns) {DropIncomingHandshake:441(11032ns) DropOutgoingHandshake:456(311ns) SwitchToState:178(91ns) ProcessPendingSessionEvents:495(13546353ns) {ForwardSessionEventToSession:612(7592782ns) ForwardSessionEventToSession:612(1013742ns) ForwardSessionEventToSession:612(12008ns) ForwardSessionEventToSession:612(4848758ns) ForwardSessionEventToSession:612(5161ns) ForwardSessionEventToSession:612(18352ns) ForwardSessionEventToSession:612(7004ns) ForwardSessionEventToSession:612(3285ns) ForwardSessionEventToSession:612(9781ns) ForwardSessionEventToSession:612(6736ns)}}}
2025-11-11T17:21:17.783275Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519153281231348:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:17.794953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519148986263984:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.795042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:17.795098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519148986263984:2048] [node 1] ICP22 created new session: [4:7571519153281231380:2048]
2025-11-11T17:21:17.795155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519153281231380:2048] [node 1] ICS09 handshake done sender: [4:7571519153281231348:2111] self: [4:7571519153281231360:0] peer: [1:7571519152978053779:0] socket: 605 qp: -1
2025-11-11T17:21:17.795197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519153281231380:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:17.795248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:17.845716Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:17.865766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519148986263986:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519153281231366:2120] held: no
2025-11-11T17:21:17.897921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519153281231379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36210
2025-11-11T17:21:17.899820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519148986263986:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(34566155ns) {HandleHandshakeStatus:395(34561812ns) {LogHandshakeFail:472(30012ns) {UpdateErrorStateLog:544(7012ns)} DropOutgoingHandshake:456(1921ns) ProcessPendingSessionEvents:495(14574255ns) {DropSessionEvent:512(31148ns) DropSessionEvent:512(12225ns) DropSessionEvent:512(7337ns) DropSessionEvent:512(5870ns) DropSessionEvent:512(5698ns) DropSessionEvent:512(6560ns) DropSessionEvent:512(6256ns) DropSessionEvent:512(4964ns) DropSessionEvent:512(33396ns) DropSessionEvent:512(6303ns)}}}
2025-11-11T17:21:17.909999Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519153281231366:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:17.914234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519148986263986:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:17.919398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:17.919459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519148986263986:2050] [node 3] ICP22 created new session: [4:7571519153281231382:2050]
2025-11-11T17:21:17.921519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519153281231382:2050] [node 3] ICS09 handshake done sender: [4:7571519153281231366:2120] self: [4:7571519153281231378:0] peer: [3:7571519152875505629:0] socket: 641 qp: -1
2025-11-11T17:21:17.922295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519153281231382:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:17.922340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:17.969782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:17.970016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519152628167708:2048] [node 1] ICS01 socket: 607 reason# EndOfStream
2025-11-11T17:21:17.970070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519148333200321:2048] [node 1] ICP30 unregister session Session# [2:7571519152628167708:2048] VirtualId# [2:7571519152628167697:0]
2025-11-11T17:21:17.970109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519152628167708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:17.970229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:17.970280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:17.967591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:17.967842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519152875505643:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-11-11T17:21:17.967982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519148580538261:2048] [node 1] ICP30 unregister session Session# [3:7571519152875505643:2048] VirtualId# [3:7571519152875505639:0]
2025-11-11T17:21:17.968024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519152875505643:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:17.968110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:17.968154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:17.977730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:17.978002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519153281231380:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-11-11T17:21:17.978103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519148986263984:2048] [node 1] ICP30 unregister session Session# [4:7571519153281231380:2048] VirtualId# [4:7571519153281231360:0]
2025-11-11T17:21:17.978156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519153281231380:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:17.978272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:17.978325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:17.981885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:17.982091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519149469628475:2048] [node 1] ICS01 socket: 644 reason# EndOfStream
2025-11-11T17:21:17.982136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519145174661085:2048] [node 1] ICP30 unregister session Session# [5:7571519149469628475:2048] VirtualId# [5:7571519149469628466:0]
2025-11-11T17:21:17.982172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519149469628475:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:17.982266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:17.982314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:17.981760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:17.982013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519151571434736:2048] [node 1] ICS07 socket disconnect 611 reason# EndOfStream
2025-11-11T17:21:17.982045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519151571434736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:17.982161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:17.982205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519151571434736:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:17.986858Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61700) Couldn't connect to any resolved address
2025-11-11T17:21:17.987106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519151571434638:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61700) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:17.987268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519151571434736:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:17.987401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519151571434638:2048] [node 1] ICP30 unregister session Session# [6:7571519151571434736:2048] VirtualId# [6:7571519151571434729:0]
2025-11-11T17:21:17.987489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:17.987531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519151571434638:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:17.987553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:18.005843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519151571434638:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:18.038954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:18.039233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519152875505649:2049] [node 2] ICS01 socket: 618 reason# EndOfStream
2025-11-11T17:21:18.039284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519148580538262:2049] [node 2] ICP30 unregister session Session# [3:7571519152875505649:2049] VirtualId# [3:7571519152875505636:0]
2025-11-11T17:21:18.039318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519152875505649:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.039401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:18.039454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:18.045174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:18.044576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:18.044802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519149469628471:2049] [node 2] ICS07 socket disconnect 648 reason# EndOfStream
2025-11-11T17:21:18.044843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519149469628471:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.044923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:18.044966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519149469628471:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:18.045828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519151571434734:2049] [node 2] ICS01 socket: 640 reason# EndOfStream
2025-11-11T17:21:18.045876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519151571434639:2049] [node 2] ICP30 unregister session Session# [6:7571519151571434734:2049] VirtualId# [6:7571519151571434720:0]
2025-11-11T17:21:18.045904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519151571434734:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.045959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:18.046006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:18.045291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:18.045455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519153281231373:2049] [node 2] ICS07 socket disconnect 616 reason# EndOfStream
2025-11-11T17:21:18.046221Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address
2025-11-11T17:21:18.045479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519153281231373:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.045548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:18.045583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519153281231373:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:18.046354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519145174661086:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:18.046421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519149469628471:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:18.046552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519145174661086:2049] [node 2] ICP30 unregister session Session# [5:7571519149469628471:2049] VirtualId# [5:7571519149469628464:0]
2025-11-11T17:21:18.047514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:18.047591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519145174661086:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:18.047638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:18.047598Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address
2025-11-11T17:21:18.047853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519148986263985:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:18.048081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519153281231373:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:18.048113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519148986263985:2049] [node 2] ICP30 unregister session Session# [4:7571519153281231373:2049] VirtualId# [4:7571519153281231358:0]
2025-11-11T17:21:18.048461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:18.048498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519148986263985:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:18.048506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:18.057618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519145174661086:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:18.058519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519148986263985:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:18.146021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:18.146283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519151571434739:2050] [node 3] ICS01 socket: 656 reason# EndOfStream
2025-11-11T17:21:18.146325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519151571434640:2050] [node 3] ICP30 unregister session Session# [6:7571519151571434739:2050] VirtualId# [6:7571519151571434728:0]
2025-11-11T17:21:18.146360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519151571434739:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.146434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:18.146477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:18.147295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:18.147490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519149469628483:2050] [node 3] ICS01 socket: 658 reason# EndOfStream
2025-11-11T17:21:18.147536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519145174661087:2050] [node 3] ICP30 unregister session Session# [5:7571519149469628483:2050] VirtualId# [5:7571519149469628478:0]
2025-11-11T17:21:18.147560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519149469628483:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.147610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:18.147644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:18.146807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:18.147014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519153281231382:2050] [node 3] ICS01 socket: 641 reason# EndOfStream
2025-11-11T17:21:18.147058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519148986263986:2050] [node 3] ICP30 unregister session Session# [4:7571519153281231382:2050] VirtualId# [4:7571519153281231378:0]
2025-11-11T17:21:18.147083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519153281231382:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.147160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:18.147205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:18.217385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:18.217632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519151571434724:2102] [node 4] ICS01 socket: 629 reason# EndOfStream
2025-11-11T17:21:18.217684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519151571434700:2102] [node 4] ICP30 unregister session Session# [6:7571519151571434724:2102] VirtualId# [6:7571519151571434718:0]
2025-11-11T17:21:18.217718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519151571434724:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.217787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:18.217824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:18.221845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:18.222126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519149469628465:2108] [node 4] ICS01 socket: 602 reason# EndOfStream
2025-11-11T17:21:18.222167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519149469628444:2108] [node 4] ICP30 unregister session Session# [5:7571519149469628465:2108] VirtualId# [5:7571519149469628452:0]
2025-11-11T17:21:18.222202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519149469628465:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.222293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:18.222347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:18.285845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:18.286119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519151571434740:2100] [node 5] ICS01 socket: 660 reason# EndOfStream
2025-11-11T17:21:18.286164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519151571434699:2100] [node 5] ICP30 unregister session Session# [6:7571519151571434740:2100] VirtualId# [6:7571519151571434732:0]
2025-11-11T17:21:18.286203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519151571434740:2100] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:18.286330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:18.286398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:18.380963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519155968596746:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:18.381032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519155968596747:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:18.400777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519154832370687:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:18.400819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519154832370688:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:18.440739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519156619979731:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:18.440783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519156619979732:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:18.447562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519154272882176:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:18.447607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519154272882177:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:18.447636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519154272882178:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:18.467319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519154280391919:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:18.467381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519154280391920:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:18.467426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519154280391921:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:18.486994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519154955646961:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:18.487056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519154955646962:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:18.487097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519154955646963:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:18.570642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519154955647022:2105] [node 5] ICP01 ready to work
2025-11-11T17:21:18.576003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519155968596809:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.624505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280391980:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
2025-11-11T17:21:18.625980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519155968596809:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(48564444ns) {Configure:104(48562119ns) {ProcessConfigured:135(67527ns) {StartInitialHandshake:157(64399ns) {DropHandshakes:469(4282ns) {DropIncomingHandshake:441(1007ns) DropOutgoingHandshake:456(1084ns)} GenerateSessionVirtualId:988(1535ns) PrepareNewSessionHandshake:476(2560ns) {SwitchToState:178(625ns)}}}}}
2025-11-11T17:21:18.647425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519156619979795:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:18.660051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519154832370687:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(43763474ns) {Configure:104(37014935ns) {ProcessConfigured:135(79692ns) {StartInitialHandshake:157(76926ns) {DropHandshakes:469(3923ns) {DropIncomingHandshake:441(864ns) DropOutgoingHandshake:456(749ns)} GenerateSessionVirtualId:988(763ns) PrepareNewSessionHandshake:476(2620ns) {SwitchToState:178(1019ns)}}}}}
2025-11-11T17:21:18.661602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519156619979796:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:18.660555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519154832370752:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:18.644860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519155968596819:2110] [node 5] ICP01 ready to work
2025-11-11T17:21:18.678215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519154272882239:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:18.648281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519155968596820:2111] [node 6] ICP01 ready to work
2025-11-11T17:21:18.660836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519154832370753:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:18.679883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519156619979797:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:18.662275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519154832370754:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:18.683143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519155968596820:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32644733ns) {Configure:104(32636913ns) {ProcessConfigured:135(70075ns) {StartInitialHandshake:157(66617ns) {DropHandshakes:469(4534ns) {DropIncomingHandshake:441(867ns) DropOutgoingHandshake:456(1183ns)} GenerateSessionVirtualId:988(892ns) PrepareNewSessionHandshake:476(1834ns) {SwitchToState:178(752ns)}}}}}
2025-11-11T17:21:18.703212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519154280391919:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39633026ns) {Configure:104(39627710ns) {ProcessConfigured:135(76091ns) {StartInitialHandshake:157(73028ns) {DropHandshakes:469(3565ns) {DropIncomingHandshake:441(876ns) DropOutgoingHandshake:456(679ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:476(2157ns) {SwitchToState:178(578ns)}}}}}
2025-11-11T17:21:18.703979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519154280391985:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:18.704337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519154280391986:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.756380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519154280391920:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(51734032ns) {Configure:104(51728456ns) {ProcessConfigured:135(73398ns) {StartInitialHandshake:157(65530ns) {DropHandshakes:469(3836ns) {DropIncomingHandshake:441(799ns) DropOutgoingHandshake:456(622ns)} GenerateSessionVirtualId:988(1049ns) PrepareNewSessionHandshake:476(3018ns) {SwitchToState:178(1127ns)}}}}}
2025-11-11T17:21:18.734547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519155968596828:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-11T17:21:18.725955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519154832370688:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(51573919ns) {Configure:104(51569615ns) {ProcessConfigured:135(866385ns) {StartInitialHandshake:157(862208ns) {DropHandshakes:469(3769ns) {DropIncomingHandshake:441(946ns) DropOutgoingHandshake:456(567ns)} GenerateSessionVirtualId:988(1053ns) PrepareNewSessionHandshake:476(4367ns) {SwitchToState:178(1441ns)}}}}}
2025-11-11T17:21:18.770793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519154272882176:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(85869441ns) {Configure:104(85865293ns) {ProcessConfigured:135(83562ns) {StartInitialHandshake:157(74503ns) {DropHandshakes:469(3133ns) {DropIncomingHandshake:441(654ns) DropOutgoingHandshake:456(721ns)} GenerateSessionVirtualId:988(1090ns) PrepareNewSessionHandshake:476(2416ns) {SwitchToState:178(858ns)}}}}}
2025-11-11T17:21:18.771220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519154272882242:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:18.761622Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519154280391980:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.766622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519156619979731:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(82155369ns) {Configure:104(82149224ns) {ProcessConfigured:135(63487ns) {StartInitialHandshake:157(60473ns) {DropHandshakes:469(8770ns) {DropIncomingHandshake:441(710ns) DropOutgoingHandshake:456(717ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:476(2129ns) {SwitchToState:178(679ns)}}}}}
2025-11-11T17:21:18.751608Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596828:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.805928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519154272882249:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41242
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.813174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519154832370757:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36947
2025-11-11T17:21:18.813388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519154832370758:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36947
2025-11-11T17:21:18.813503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519154832370759:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36947
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.818836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519154955646963:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(63595211ns) {Configure:104(63584054ns) {ProcessConfigured:135(47126ns) {StartInitialHandshake:157(44411ns) {DropHandshakes:469(1518ns) {DropIncomingHandshake:441(390ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(2855ns) {SwitchToState:178(784ns)}}}}}
2025-11-11T17:21:18.819236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519154955647036:2112] [node 4] ICP01 ready to work
2025-11-11T17:21:18.821472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519156619979796:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(44539193ns) {Configure:104(44529125ns) {ProcessConfigured:135(3116796ns) {StartInitialHandshake:157(3114278ns) {DropHandshakes:469(3346ns) {DropIncomingHandshake:441(738ns) DropOutgoingHandshake:456(831ns)} GenerateSessionVirtualId:988(1128ns) PrepareNewSessionHandshake:476(3658ns) {SwitchToState:178(1311ns)}}}}}
2025-11-11T17:21:18.820269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280391999:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.833315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519156619979806:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20787
2025-11-11T17:21:18.820596Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519154832370759:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.837823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519154832370758:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.843818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519156619979807:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20787
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.844355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519156619979808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20787
2025-11-11T17:21:18.844432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519156619979809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20787
2025-11-11T17:21:18.844685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519156619979806:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.849708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519156619979731:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.850645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647039:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:18.851112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519156619979807:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.854192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280392000:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
2025-11-11T17:21:18.849022Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519154272882249:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.864245Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519154280392000:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.875459Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519154955647039:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.875584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519154955646963:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.850132Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519154832370757:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.849865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519155968596830:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-11T17:21:18.862774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519154832370687:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.874111Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519156619979809:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.874970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519156619979732:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.875063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519156619979817:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20787
2025-11-11T17:21:18.869965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519154272882255:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41242
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.865573Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596811:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.865623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519155968596811:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.866283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519155968596831:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-11T17:21:18.885673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519156619979803:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.885719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519156619979803:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.886274Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519156619979808:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.881948Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519154832370761:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.866359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519155968596832:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.909389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647041:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:18.870156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519154272882176:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.881989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519154832370761:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.884177Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519154280391999:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.884322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519154280391921:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.884565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519154280391985:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.890499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280392003:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
2025-11-11T17:21:18.878113Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519154272882255:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.882799Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519154832370756:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.882820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519154832370756:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.890978Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519154832370761:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:18.902703Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519154832370756:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:18.934035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519154832370752:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.934122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:18.934179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519154832370752:2107] [node 4] ICP22 created new session: [2:7571519154832370774:2107]
2025-11-11T17:21:18.934237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519154832370774:2107] [node 4] ICS09 handshake done sender: [2:7571519154832370761:2115] self: [2:7571519154832370760:0] peer: [4:7571519154272882257:0] socket: 628 qp: -1
2025-11-11T17:21:18.934282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519154832370774:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:18.934327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:18.866558Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596830:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.935761Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519154955647041:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.935940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519154955647036:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.878365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519154272882177:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.937231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519154832370688:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.937286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:18.937333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519154832370688:2049] [node 3] ICP22 created new session: [2:7571519154832370776:2049]
2025-11-11T17:21:18.937378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519154832370776:2049] [node 3] ICS09 handshake done sender: [2:7571519154832370756:2111] self: [2:7571519154832370755:0] peer: [3:7571519156619979818:0] socket: 627 qp: -1
2025-11-11T17:21:18.937413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519154832370776:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:18.937448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:18.938672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519154832370778:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36947
2025-11-11T17:21:18.867009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519155968596833:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.876523Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596808:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.941983Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519156619979801:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.886595Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519154272882246:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.948307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647043:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:18.947908Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519154832370757:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:18.942026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519156619979801:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.886616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519154272882246:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.876552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519155968596808:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.942632Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:18.952329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519156619979731:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519156619979806:2115] held: no
2025-11-11T17:21:18.949998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519154832370687:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.877234Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519155968596811:2109] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:18.950055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:18.950105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519154832370687:2048] [node 1] ICP22 created new session: [2:7571519154832370781:2048]
2025-11-11T17:21:18.950456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519154832370781:2048] [node 1] ICS09 handshake done sender: [2:7571519154832370757:2112] self: [2:7571519154832370771:0] peer: [1:7571519155968596807:0] socket: 619 qp: -1
2025-11-11T17:21:18.950679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519154832370781:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:18.950721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:18.887598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519154272882258:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41242
2025-11-11T17:21:18.906151Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519154272882255:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:18.906527Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519154272882246:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:18.906790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519154272882177:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.906875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:18.906939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519154272882177:2049] [node 2] ICP22 created new session: [4:7571519154272882260:2049]
2025-11-11T17:21:18.906994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519154272882260:2049] [node 2] ICS09 handshake done sender: [4:7571519154272882255:2116] self: [4:7571519154272882257:0] peer: [2:7571519154832370760:0] socket: 629 qp: -1
2025-11-11T17:21:18.907029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519154272882260:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:18.907071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:18.907577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519154272882239:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.907636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:18.907676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519154272882239:2108] [node 5] ICP22 created new session: [4:7571519154272882262:2108]
2025-11-11T17:21:18.907724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519154272882262:2108] [node 5] ICS09 handshake done sender: [4:7571519154272882246:2112] self: [4:7571519154272882245:0] peer: [5:7571519154280392002:0] socket: 623 qp: -1
2025-11-11T17:21:18.907758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519154272882262:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:18.907788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:18.922445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519154272882265:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41242
2025-11-11T17:21:18.948570Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519154272882251:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.948622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519154272882251:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.953881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519154272882249:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:18.954230Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519154272882251:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.955652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647044:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:18.956121Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519154955647041:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:18.956307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519154272882176:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.892260Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596818:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.956360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:18.892287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519155968596818:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.914275Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596832:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.914369Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596833:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.914488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519155968596747:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.914534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:18.914585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519155968596747:2049] [node 3] ICP22 created new session: [1:7571519155968596838:2049]
2025-11-11T17:21:18.914632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519155968596838:2049] [node 3] ICS09 handshake done sender: [1:7571519155968596811:2109] self: [1:7571519155968596810:0] peer: [3:7571519156619979815:0] socket: 609 qp: -1
2025-11-11T17:21:18.914667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519155968596838:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:18.914702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:18.956668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519154272882176:2048] [node 1] ICP22 created new session: [4:7571519154272882267:2048]
2025-11-11T17:21:18.957136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519154272882267:2048] [node 1] ICS09 handshake done sender: [4:7571519154272882249:2114] self: [4:7571519154272882256:0] peer: [1:7571519155968596817:0] socket: 618 qp: -1
2025-11-11T17:21:18.957166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519154272882267:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:18.957198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:18.957533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519154272882242:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.957557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:18.957606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519154272882242:2109] [node 6] ICP22 created new session: [4:7571519154272882269:2109]
2025-11-11T17:21:18.957645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519154272882269:2109] [node 6] ICS09 handshake done sender: [4:7571519154272882251:2115] self: [4:7571519154272882250:0] peer: [6:7571519154955647042:0] socket: 649 qp: -1
2025-11-11T17:21:18.957673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519154272882269:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:18.957704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:18.915232Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596831:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.956568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519154955647036:2112] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.915840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519155968596747:2049] [node 3] ICP18 handshake (actor [1:7571519155968596833:2122]) is too old
2025-11-11T17:21:18.959395Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519156619979806:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:18.958920Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519154280391999:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:18.936575Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519155968596808:2108] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:18.951240Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519155968596818:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:18.953111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519155968596746:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.954336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:18.954402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519155968596746:2048] [node 2] ICP22 created new session: [1:7571519155968596843:2048]
2025-11-11T17:21:18.954464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519155968596843:2048] [node 2] ICS09 handshake done sender: [1:7571519155968596808:2108] self: [1:7571519155968596807:0] peer: [2:7571519154832370771:0] socket: 608 qp: -1
2025-11-11T17:21:18.954496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519155968596843:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:18.956455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:18.961100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519155968596809:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.961204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:18.961245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519155968596809:2107] [node 4] ICP22 created new session: [1:7571519155968596845:2107]
2025-11-11T17:21:18.961303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519155968596845:2107] [node 4] ICS09 handshake done sender: [1:7571519155968596818:2115] self: [1:7571519155968596817:0] peer: [4:7571519154272882256:0] socket: 610 qp: -1
2025-11-11T17:21:18.961333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519155968596845:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:18.961360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:18.956607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:18.957059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519154955647036:2112] [node 4] ICP22 created new session: [6:7571519154955647045:2112]
2025-11-11T17:21:18.959265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519154955647045:2112] [node 4] ICS09 handshake done sender: [6:7571519154955647041:2116] self: [6:7571519154955647042:0] peer: [4:7571519154272882250:0] socket: 650 qp: -1
2025-11-11T17:21:18.959310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519154955647045:2112] [node 4] ICS10 traffic start
2025-11-11T17:21:18.959341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.963577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519156619979731:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.963626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:18.963949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519156619979731:2048] [node 1] ICP22 created new session: [3:7571519156619979823:2048]
2025-11-11T17:21:18.964571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519156619979823:2048] [node 1] ICS09 handshake done sender: [3:7571519156619979806:2115] self: [3:7571519156619979815:0] peer: [1:7571519155968596810:0] socket: 630 qp: -1
2025-11-11T17:21:18.964614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519156619979823:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:18.965787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519154280391985:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.965836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:18.965890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519154280391985:2109] [node 4] ICP22 created new session: [5:7571519154280392007:2109]
2025-11-11T17:21:18.965938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519154280392007:2109] [node 4] ICS09 handshake done sender: [5:7571519154280391999:2116] self: [5:7571519154280392002:0] peer: [4:7571519154272882245:0] socket: 625 qp: -1
2025-11-11T17:21:18.965967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519154280392007:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:18.965996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:18.964651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.965009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519156619979819:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20787
2025-11-11T17:21:18.965618Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519156619979803:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.968442Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519156619979801:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:18.967322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280392004:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
2025-11-11T17:21:18.967530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280392005:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
2025-11-11T17:21:18.968161Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519154280392005:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.968287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280392006:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.971326Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519154955647043:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.971730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647047:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.972186Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519154280392000:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:18.975218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519154955646962:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.975369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647048:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:18.975610Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519154955647039:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:18.975274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519156619979797:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.975370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:18.972988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519154280391920:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.981223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519154955646963:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.975433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519156619979797:2109] [node 6] ICP22 created new session: [3:7571519156619979826:2109]
2025-11-11T17:21:18.973113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519154280391921:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:18.977853Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519154832370763:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.975748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519156619979826:2109] [node 6] ICS09 handshake done sender: [3:7571519156619979803:2113] self: [3:7571519156619979802:0] peer: [6:7571519154955647040:0] socket: 638 qp: -1
2025-11-11T17:21:18.982939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:18.973157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:18.973203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519154280391921:2050] [node 3] ICP22 created new session: [5:7571519154280392011:2050]
2025-11-11T17:21:18.973246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519154280392011:2050] [node 3] ICS09 handshake done sender: [5:7571519154280392000:2117] self: [5:7571519154280392001:0] peer: [3:7571519156619979800:0] socket: 641 qp: -1
2025-11-11T17:21:18.973279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519154280392011:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:18.973316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:18.988206Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519154280392004:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.988617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519154280391919:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:18.983002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519154955646963:2050] [node 3] ICP22 created new session: [6:7571519154955647051:2050]
2025-11-11T17:21:18.986766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519154955647051:2050] [node 3] ICS09 handshake done sender: [6:7571519154955647039:2115] self: [6:7571519154955647040:0] peer: [3:7571519156619979802:0] socket: 640 qp: -1
2025-11-11T17:21:18.987671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519154955647051:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:18.987720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:18.977895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519154832370763:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.976000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519156619979826:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:18.976045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:18.982381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519156619979796:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:18.985441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:18.985500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519156619979796:2108] [node 5] ICP22 created new session: [3:7571519156619979828:2108]
2025-11-11T17:21:18.985550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519156619979828:2108] [node 5] ICS09 handshake done sender: [3:7571519156619979801:2112] self: [3:7571519156619979800:0] peer: [5:7571519154280392001:0] socket: 637 qp: -1
2025-11-11T17:21:18.985603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519156619979828:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:18.985644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:18.986536Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519156619979809:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:18.989290Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519154832370765:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.989312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519154832370765:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:18.989222Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596823:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:18.989267Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519155968596823:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.992639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519154832370783:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36947
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.992797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519154832370763:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:18.994075Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519155968596823:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:19.003568Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519154832370765:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:19.001240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647053:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:19.001371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519154955647054:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:19.004685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519154955647048:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.003904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280392014:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
2025-11-11T17:21:19.005376Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519154955647043:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:19.006274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519154832370753:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.006383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:19.006433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519154832370753:2108] [node 5] ICP22 created new session: [2:7571519159127338082:2108]
2025-11-11T17:21:19.006488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519159127338082:2108] [node 5] ICS09 handshake done sender: [2:7571519154832370763:2116] self: [2:7571519154832370762:0] peer: [5:7571519154280392010:0] socket: 655 qp: -1
2025-11-11T17:21:19.006527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519159127338082:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.006566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:19.009859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519154280392015:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21272
2025-11-11T17:21:19.007635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519154955647022:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.007704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519154955646962:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.008035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:19.008088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519154955646962:2049] [node 2] ICP22 created new session: [6:7571519159250614352:2049]
2025-11-11T17:21:19.008132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519159250614352:2049] [node 2] ICS09 handshake done sender: [6:7571519154955647043:2117] self: [6:7571519154955647050:0] peer: [2:7571519154832370764:0] socket: 657 qp: -1
2025-11-11T17:21:19.008171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519159250614352:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:19.008397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:19.006975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519154832370754:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.008054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:19.008095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519154832370754:2109] [node 6] ICP22 created new session: [2:7571519159127338084:2109]
2025-11-11T17:21:19.008153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519159127338084:2109] [node 6] ICS09 handshake done sender: [2:7571519154832370765:2117] self: [2:7571519154832370764:0] peer: [6:7571519154955647050:0] socket: 656 qp: -1
2025-11-11T17:21:19.008182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519159127338084:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:19.008234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:19.013398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519156619979732:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.013457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:19.013503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519156619979732:2049] [node 2] ICP22 created new session: [3:7571519160914947127:2049]
2025-11-11T17:21:19.013544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519160914947127:2049] [node 2] ICS09 handshake done sender: [3:7571519156619979809:2118] self: [3:7571519156619979818:0] peer: [2:7571519154832370755:0] socket: 633 qp: -1
2025-11-11T17:21:19.013582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519160914947127:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:19.013661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:19.009775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519155968596819:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.011134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:19.011194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519155968596819:2110] [node 5] ICP22 created new session: [1:7571519160263564145:2110]
2025-11-11T17:21:19.011863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519160263564145:2110] [node 5] ICS09 handshake done sender: [1:7571519155968596823:2116] self: [1:7571519155968596822:0] peer: [5:7571519154280392013:0] socket: 653 qp: -1
2025-11-11T17:21:19.011895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519160263564145:2110] [node 5] ICS10 traffic start
2025-11-11T17:21:19.011933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:19.012696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519154955647053:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.019427Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519154280391994:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.019474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519154280391994:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.030654Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.042958Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:19.045519Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519154280392004:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:19.051225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519155968596825:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.051265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519155968596825:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.053571Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519155968596825:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.054566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158567849567:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41242
2025-11-11T17:21:19.054062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519160914947126:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20787
2025-11-11T17:21:19.050769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519154955646961:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.054152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519159250614355:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:19.054022Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.055032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519155968596820:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.054224Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:19.055471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519156619979797:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:19.056725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:19.056788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519155968596820:2111] [node 6] ICP22 created new session: [1:7571519160263564148:2111]
2025-11-11T17:21:19.056839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519160263564148:2111] [node 6] ICS09 handshake done sender: [1:7571519155968596825:2117] self: [1:7571519155968596824:0] peer: [6:7571519159250614354:0] socket: 616 qp: -1
2025-11-11T17:21:19.056874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519160263564148:2111] [node 6] ICS10 traffic start
2025-11-11T17:21:19.056908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:19.059552Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.060940Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158567849567:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.061071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519154272882178:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.058655Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519154280392005:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:19.058855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519154280391920:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519154280392005:2120] held: no
2025-11-11T17:21:19.059522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519154280391920:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.060616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:19.060661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519154280391920:2049] [node 2] ICP22 created new session: [5:7571519158575359313:2049]
2025-11-11T17:21:19.061345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519158575359313:2049] [node 2] ICS09 handshake done sender: [5:7571519154280392005:2120] self: [5:7571519154280392010:0] peer: [2:7571519154832370762:0] socket: 636 qp: -1
2025-11-11T17:21:19.061357Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519156619979811:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.061882Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519154955647053:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:19.061372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519158575359313:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:19.061395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519156619979811:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.061399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:19.064924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519154955646961:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519154955647053:2121] held: no
2025-11-11T17:21:19.065054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519154955646961:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.065099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:19.065157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519154955646961:2048] [node 1] ICP22 created new session: [6:7571519159250614356:2048]
2025-11-11T17:21:19.065210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519159250614356:2048] [node 1] ICS09 handshake done sender: [6:7571519154955647053:2121] self: [6:7571519159250614354:0] peer: [1:7571519155968596824:0] socket: 672 qp: -1
2025-11-11T17:21:19.065245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519159250614356:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:19.065317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:19.066500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519154280391919:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.066554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:19.066585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519154280391919:2048] [node 1] ICP22 created new session: [5:7571519158575359315:2048]
2025-11-11T17:21:19.067262Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519156619979811:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.068440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519159250614358:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22618
2025-11-11T17:21:19.068910Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519154955647048:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:19.069136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519154955647022:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.069190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:19.069232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519154955647022:2105] [node 5] ICP22 created new session: [6:7571519159250614359:2105]
2025-11-11T17:21:19.069287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519159250614359:2105] [node 5] ICS09 handshake done sender: [6:7571519154955647048:2120] self: [6:7571519159250614351:0] peer: [5:7571519154280391993:0] socket: 665 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.066616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519158575359315:2048] [node 1] ICS09 handshake done sender: [5:7571519154280392004:2119] self: [5:7571519154280392013:0] peer: [1:7571519155968596822:0] socket: 659 qp: -1
2025-11-11T17:21:19.069316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519159250614359:2105] [node 5] ICS10 traffic start
2025-11-11T17:21:19.067799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519156619979795:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.069344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:19.066639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519158575359315:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:19.067859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:19.067904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519156619979795:2107] [node 4] ICP22 created new session: [3:7571519160914947131:2107]
2025-11-11T17:21:19.067955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519160914947131:2107] [node 4] ICS09 handshake done sender: [3:7571519156619979811:2119] self: [3:7571519156619979810:0] peer: [4:7571519158567849568:0] socket: 612 qp: -1
2025-11-11T17:21:19.067987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519160914947131:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:19.068022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:19.071852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158567849569:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41242
2025-11-11T17:21:19.066661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:19.072710Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519158567849567:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:19.066834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519154280391919:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:19.067910Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519154280391994:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:19.072010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519154280391986:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.072129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:19.072378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519154280391986:2111] [node 6] ICP22 created new session: [5:7571519158575359317:2111]
2025-11-11T17:21:19.072672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519158575359317:2111] [node 6] ICS09 handshake done sender: [5:7571519154280391994:2115] self: [5:7571519154280391993:0] peer: [6:7571519159250614351:0] socket: 664 qp: -1
2025-11-11T17:21:19.072703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519158575359317:2111] [node 6] ICS10 traffic start
2025-11-11T17:21:19.072740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:19.073353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519154272882178:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.073456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:19.073503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519154272882178:2050] [node 3] ICP22 created new session: [4:7571519158567849570:2050]
2025-11-11T17:21:19.073548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519158567849570:2050] [node 3] ICS09 handshake done sender: [4:7571519158567849567:2119] self: [4:7571519158567849568:0] peer: [3:7571519156619979810:0] socket: 615 qp: -1
2025-11-11T17:21:19.073580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519158567849570:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:19.073628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:19.074334Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13137) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.075560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519154272882178:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:19.098377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:19.098636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519154832370781:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-11-11T17:21:19.098673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519154832370687:2048] [node 1] ICP30 unregister session Session# [2:7571519154832370781:2048] VirtualId# [2:7571519154832370771:0]
2025-11-11T17:21:19.098709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519154832370781:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.098836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:19.098906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:19.099600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:19.099844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519156619979823:2048] [node 1] ICS01 socket: 630 reason# EndOfStream
2025-11-11T17:21:19.099970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519156619979731:2048] [node 1] ICP30 unregister session Session# [3:7571519156619979823:2048] VirtualId# [3:7571519156619979815:0]
2025-11-11T17:21:19.100003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519156619979823:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.100081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:19.100120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:19.102638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:19.102870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519154272882267:2048] [node 1] ICS01 socket: 618 reason# EndOfStream
2025-11-11T17:21:19.102905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519154272882176:2048] [node 1] ICP30 unregister session Session# [4:7571519154272882267:2048] VirtualId# [4:7571519154272882256:0]
2025-11-11T17:21:19.102937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519154272882267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.103013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:19.103051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:19.103723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:19.103909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519158575359315:2048] [node 1] ICS01 socket: 659 reason# EndOfStream
2025-11-11T17:21:19.103947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519154280391919:2048] [node 1] ICP30 unregister session Session# [5:7571519158575359315:2048] VirtualId# [5:7571519154280392013:0]
2025-11-11T17:21:19.103972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519158575359315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.104032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:19.104064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:19.104822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:19.105029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519159250614356:2048] [node 1] ICS01 socket: 672 reason# EndOfStream
2025-11-11T17:21:19.105075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519154955646961:2048] [node 1] ICP30 unregister session Session# [6:7571519159250614356:2048] VirtualId# [6:7571519159250614354:0]
2025-11-11T17:21:19.105102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519159250614356:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.105171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:19.105211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:19.155146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:19.155429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519160914947127:2049] [node 2] ICS07 socket disconnect 633 reason# EndOfStream
2025-11-11T17:21:19.155472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519160914947127:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.155600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:19.155656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519160914947127:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:19.156668Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21392) Couldn't connect to any resolved address
2025-11-11T17:21:19.156974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519156619979732:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21392) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:19.157036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519160914947127:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:19.157073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519156619979732:2049] [node 2] ICP30 unregister session Session# [3:7571519160914947127:2049] VirtualId# [3:7571519156619979818:0]
2025-11-11T17:21:19.157118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:19.157150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519156619979732:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21392) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:19.156863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:19.157167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21392) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:19.157094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519154272882260:2049] [node 2] ICS01 socket: 629 reason# EndOfStream
2025-11-11T17:21:19.157160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519154272882177:2049] [node 2] ICP30 unregister session Session# [4:7571519154272882260:2049] VirtualId# [4:7571519154272882257:0]
2025-11-11T17:21:19.157194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519154272882260:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.157296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:19.157358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:19.161760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:19.162056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519158575359313:2049] [node 2] ICS01 socket: 636 reason# EndOfStream
2025-11-11T17:21:19.162099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519154280391920:2049] [node 2] ICP30 unregister session Session# [5:7571519158575359313:2049] VirtualId# [5:7571519154280392010:0]
2025-11-11T17:21:19.162138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519158575359313:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.162218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:19.162277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:19.161761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:19.162486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519159250614352:2049] [node 2] ICS01 socket: 657 reason# EndOfStream
2025-11-11T17:21:19.162543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519154955646962:2049] [node 2] ICP30 unregister session Session# [6:7571519159250614352:2049] VirtualId# [6:7571519154955647050:0]
2025-11-11T17:21:19.162574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519159250614352:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.162646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:19.162698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:19.167739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519156619979732:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:19.196136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:19.196386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519158567849570:2050] [node 3] ICS07 socket disconnect 615 reason# EndOfStream
2025-11-11T17:21:19.196433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519158567849570:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.196518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:19.196572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519158567849570:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:19.197269Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13137) Couldn't connect to any resolved address
2025-11-11T17:21:19.197517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519154272882178:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13137) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:19.197607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519158567849570:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:19.197660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519154272882178:2050] [node 3] ICP30 unregister session Session# [4:7571519158567849570:2050] VirtualId# [4:7571519158567849568:0]
2025-11-11T17:21:19.197714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:19.197745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519154272882178:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:19.197760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:19.198747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:19.198959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519154280392011:2050] [node 3] ICS01 socket: 641 reason# EndOfStream
2025-11-11T17:21:19.199010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519154280391921:2050] [node 3] ICP30 unregister session Session# [5:7571519154280392011:2050] VirtualId# [5:7571519154280392001:0]
2025-11-11T17:21:19.199052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519154280392011:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.199135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:19.199179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:19.199282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:19.199524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519154955647051:2050] [node 3] ICS01 socket: 640 reason# EndOfStream
2025-11-11T17:21:19.199560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519154955646963:2050] [node 3] ICP30 unregister session Session# [6:7571519154955647051:2050] VirtualId# [6:7571519154955647040:0]
2025-11-11T17:21:19.199588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519154955647051:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.199654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:19.199703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:19.208631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519154272882178:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:19.248093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:19.248344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519154280392007:2109] [node 4] ICS07 socket disconnect 625 reason# EndOfStream
2025-11-11T17:21:19.248385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519154280392007:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.248523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:19.248582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519154280392007:2109] [node 4] ICS15 start handshake
2025-11-11T17:21:19.249307Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6817) Couldn't connect to any resolved address
2025-11-11T17:21:19.249812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519154280391985:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6817) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:19.249869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519154280392007:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:19.249912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519154280391985:2109] [node 4] ICP30 unregister session Session# [5:7571519154280392007:2109] VirtualId# [5:7571519154280392002:0]
2025-11-11T17:21:19.249965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:19.249994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519154280391985:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:19.250007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:19.253760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:19.254092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519154955647045:2112] [node 4] ICS01 socket: 650 reason# EndOfStream
2025-11-11T17:21:19.254163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519154955647036:2112] [node 4] ICP30 unregister session Session# [6:7571519154955647045:2112] VirtualId# [6:7571519154955647042:0]
2025-11-11T17:21:19.254219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519154955647045:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.254345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:19.254400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:19.279955Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:19.280187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519159250614359:2105] [node 5] ICS01 socket: 665 reason# EndOfStream
2025-11-11T17:21:19.280228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519154955647022:2105] [node 5] ICP30 unregister session Session# [6:7571519159250614359:2105] VirtualId# [6:7571519159250614351:0]
2025-11-11T17:21:19.280262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519159250614359:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:19.280368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:19.280414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:19.368547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519161812337966:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:19.368596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519161812337967:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:19.390096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519161855439362:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:19.390159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519161855439363:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:19.412129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519158649075680:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:19.412180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519158649075681:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:19.433694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519158669049682:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:19.433743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519158669049683:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:19.434482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519158669049684:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:19.455890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519159292588361:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:19.455941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519159292588362:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:19.456022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519159292588363:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:19.476927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519160607549937:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:19.476993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519160607549939:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:19.477033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519160607549938:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:19.568783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519160607549983:2094] [node 4] ICP01 ready to work
2025-11-11T17:21:19.626388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519158669049745:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:19.626265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519161855439425:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:19.626361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519158649075743:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:19.610403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519160607549992:2091] [node 5] ICP01 ready to work
2025-11-11T17:21:19.629522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519161812338029:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:19.629620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519158649075744:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:19.640536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519159292588424:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:19.626399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519161855439426:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:19.640905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519161812338030:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:19.639478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519158669049746:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:19.675660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519158669049683:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(49014593ns) {Configure:104(49010560ns) {ProcessConfigured:135(61924ns) {StartInitialHandshake:157(59946ns) {DropHandshakes:469(3695ns) {DropIncomingHandshake:441(492ns) DropOutgoingHandshake:456(261ns)} GenerateSessionVirtualId:988(937ns) PrepareNewSessionHandshake:476(1287ns) {SwitchToState:178(540ns)}}}}}
2025-11-11T17:21:19.629521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519161855439427:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:19.654433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519158649075745:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:19.660395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519159292588425:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:19.653020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519161812338031:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:19.671347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519161812337967:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(111952275ns) {Configure:104(102400019ns) {ProcessConfigured:135(78346ns) {StartInitialHandshake:157(75014ns) {DropHandshakes:469(2873ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(791ns)} GenerateSessionVirtualId:988(874ns) PrepareNewSessionHandshake:476(2115ns) {SwitchToState:178(777ns)}}}}}
2025-11-11T17:21:19.695942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519161855439362:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(69267698ns) {Configure:104(69262106ns) {ProcessConfigured:135(89882ns) {StartInitialHandshake:157(83245ns) {DropHandshakes:469(2958ns) {DropIncomingHandshake:441(593ns) DropOutgoingHandshake:456(497ns)} GenerateSessionVirtualId:988(1421ns) PrepareNewSessionHandshake:476(2043ns) {SwitchToState:178(921ns)}}}}}
2025-11-11T17:21:19.702697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519160607549983:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(120334046ns) {Configure:104(108901919ns) {ProcessConfigured:135(77081ns) {StartInitialHandshake:157(74293ns) {DropHandshakes:469(2343ns) {DropIncomingHandshake:441(702ns) DropOutgoingHandshake:456(409ns)} GenerateSessionVirtualId:988(1513ns) PrepareNewSessionHandshake:476(2141ns) {SwitchToState:178(697ns)}}}}}
2025-11-11T17:21:19.719100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519158649075680:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(66110523ns) {Configure:104(66103652ns) {ProcessConfigured:135(76363ns) {StartInitialHandshake:157(73121ns) {DropHandshakes:469(2973ns) {DropIncomingHandshake:441(389ns) DropOutgoingHandshake:456(668ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:476(1680ns) {SwitchToState:178(614ns)}}}}}
2025-11-11T17:21:19.723270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519158669049745:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(47384698ns) {Configure:104(47379240ns) {ProcessConfigured:135(71117ns) {StartInitialHandshake:157(62919ns) {DropHandshakes:469(3546ns) {DropIncomingHandshake:441(1085ns) DropOutgoingHandshake:456(541ns)} GenerateSessionVirtualId:988(1062ns) PrepareNewSessionHandshake:476(2550ns) {SwitchToState:178(861ns)}}}}}
2025-11-11T17:21:19.731050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519158669049684:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(64600039ns) {Configure:104(64597201ns) {ProcessConfigured:135(26603773ns) {StartInitialHandshake:157(26598679ns) {DropHandshakes:469(2473ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:476(5340ns) {SwitchToState:178(2484ns)}}}}}
2025-11-11T17:21:19.737032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519159292588361:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(85446643ns) {Configure:104(85433329ns) {ProcessConfigured:135(81309ns) {StartInitialHandshake:157(78510ns) {DropHandshakes:469(2316ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(457ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:476(1541ns) {SwitchToState:178(495ns)}}}}}
2025-11-11T17:21:19.741898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519160607549992:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31952050ns) {Configure:104(31942144ns) {ProcessConfigured:135(10798885ns) {StartInitialHandshake:157(10796926ns) {DropHandshakes:469(5519ns) {DropIncomingHandshake:441(537ns) DropOutgoingHandshake:456(442ns)} GenerateSessionVirtualId:988(889ns) PrepareNewSessionHandshake:476(3444ns) {SwitchToState:178(520ns)}}}}}
2025-11-11T17:21:19.726876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519161855439425:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30621138ns) {Configure:104(30615760ns) {ProcessConfigured:135(3123534ns) {StartInitialHandshake:157(3120765ns) {DropHandshakes:469(2609ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(478ns)} GenerateSessionVirtualId:988(939ns) PrepareNewSessionHandshake:476(3466ns) {SwitchToState:178(1363ns)}}}}}
2025-11-11T17:21:19.754530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519161812337966:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(80916662ns) {Configure:104(80906466ns) {ProcessConfigured:135(44296ns) {StartInitialHandshake:157(43793ns) {DropHandshakes:469(414ns) {DropIncomingHandshake:441(193ns) DropOutgoingHandshake:456(108ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:476(1025ns) {SwitchToState:178(382ns)}}}}}
2025-11-11T17:21:19.760204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519161855439426:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33045846ns) {Configure:104(33041222ns) {ProcessConfigured:135(3778445ns) {StartInitialHandshake:157(3775792ns) {DropHandshakes:469(2120ns) {DropIncomingHandshake:441(462ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:476(3306ns) {SwitchToState:178(1743ns)}}}}}
2025-11-11T17:21:19.762058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519159292588363:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(95188717ns) {Configure:104(95183524ns) {ProcessConfigured:135(54724ns) {StartInitialHandshake:157(50725ns) {DropHandshakes:469(2436ns) {DropIncomingHandshake:441(509ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:476(1177ns) {SwitchToState:178(550ns)}}}}}
2025-11-11T17:21:19.762734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519158649075681:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(92147028ns) {Configure:104(92135387ns) {ProcessConfigured:135(3023732ns) {StartInitialHandshake:157(3021150ns) {DropHandshakes:469(1413ns) {DropIncomingHandshake:441(304ns) DropOutgoingHandshake:456(259ns)} GenerateSessionVirtualId:988(647ns) PrepareNewSessionHandshake:476(3640ns) {SwitchToState:178(1511ns)}}}}}
2025-11-11T17:21:19.764224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519161855439363:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(121014124ns) {Configure:104(121007122ns) {ProcessConfigured:135(7017822ns) {StartInitialHandshake:157(3830066ns) {DropHandshakes:469(1270ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(991ns) PrepareNewSessionHandshake:476(3594ns) {SwitchToState:178(2033ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.764230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519158649075743:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(41478711ns) {Configure:104(41473647ns) {ProcessConfigured:135(6446129ns) {StartInitialHandshake:157(6444286ns) {DropHandshakes:469(932ns) {DropIncomingHandshake:441(237ns) DropOutgoingHandshake:456(337ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:476(2693ns) {SwitchToState:178(1268ns)}}}}}
2025-11-11T17:21:19.788743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519160607549937:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(108984647ns) {Configure:104(108977373ns) {ProcessConfigured:135(8686764ns) {StartInitialHandshake:157(8683969ns) {DropHandshakes:469(2008ns) {DropIncomingHandshake:441(504ns) DropOutgoingHandshake:456(483ns)} GenerateSessionVirtualId:988(710ns) PrepareNewSessionHandshake:476(3197ns) {SwitchToState:178(1217ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.789724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519161855439438:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-11T17:21:19.797067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519161812338031:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(42301086ns) {Configure:104(42297829ns) {ProcessConfigured:135(71096ns) {StartInitialHandshake:157(66890ns) {DropHandshakes:469(2848ns) {DropIncomingHandshake:441(660ns) DropOutgoingHandshake:456(457ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:476(1775ns) {SwitchToState:178(529ns)}}}}}
2025-11-11T17:21:19.794929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049760:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.802490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519161812338030:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(58897551ns) {Configure:104(58891451ns) {ProcessConfigured:135(3926937ns) {StartInitialHandshake:157(3923561ns) {DropHandshakes:469(3871ns) {DropIncomingHandshake:441(863ns) DropOutgoingHandshake:456(542ns)} GenerateSessionVirtualId:988(1128ns) PrepareNewSessionHandshake:476(3385ns) {SwitchToState:178(951ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.795070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049762:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
2025-11-11T17:21:19.806081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519161812338043:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
2025-11-11T17:21:19.806255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519161812338044:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.805268Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158669049760:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.809116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519158649075745:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23224121ns) {Configure:104(23219154ns) {ProcessConfigured:135(170845ns) {StartInitialHandshake:157(167917ns) {DropHandshakes:469(9225ns) {DropIncomingHandshake:441(440ns) DropOutgoingHandshake:456(387ns)} GenerateSessionVirtualId:988(1206ns) PrepareNewSessionHandshake:476(2168ns) {SwitchToState:178(470ns)}}}}}
2025-11-11T17:21:19.807209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049764:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
2025-11-11T17:21:19.815877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519161812338042:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
2025-11-11T17:21:19.817860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519159292588439:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:19.817898Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439438:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.813100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519158649075752:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35934
2025-11-11T17:21:19.811164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049761:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
2025-11-11T17:21:19.816115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519161812338048:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
2025-11-11T17:21:19.814197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
2025-11-11T17:21:19.818227Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158669049761:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.813228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519158649075753:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35934
2025-11-11T17:21:19.813616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519158649075756:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35934
2025-11-11T17:21:19.817695Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075753:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.817867Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075752:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.825310Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075756:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.832965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519159292588440:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.818322Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338043:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.830948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519158649075680:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.833472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519158649075762:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35934
2025-11-11T17:21:19.834216Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519158649075756:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:19.834407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519158649075680:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.834457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:19.834509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519158649075680:2048] [node 1] ICP22 created new session: [3:7571519158649075763:2048]
2025-11-11T17:21:19.834557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519158649075763:2048] [node 1] ICS09 handshake done sender: [3:7571519158649075756:2116] self: [3:7571519158649075761:0] peer: [1:7571519161812338032:0] socket: 640 qp: -1
2025-11-11T17:21:19.834591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519158649075763:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:19.834624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:19.835363Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158669049762:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.835499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519158669049683:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.835375Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519159292588439:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.820151Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338042:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.837059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519159292588362:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.840093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519158649075744:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(75563625ns) {Configure:104(75556396ns) {ProcessConfigured:135(4785292ns) {StartInitialHandshake:157(4783147ns) {DropHandshakes:469(1397ns) {DropIncomingHandshake:441(295ns) DropOutgoingHandshake:456(296ns)} GenerateSessionVirtualId:988(456ns) PrepareNewSessionHandshake:476(2548ns) {SwitchToState:178(421ns)}}}}}
2025-11-11T17:21:19.840173Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439434:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.822519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338044:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.842964Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158669049764:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.849694Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519159292588440:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.849855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519159292588361:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.853568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519159292588443:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:19.853641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519159292588444:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:19.830973Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338048:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.840223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519161855439434:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.844533Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158669049765:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.860393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075751:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.851738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519161855439442:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-11T17:21:19.831190Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338033:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.844640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519158669049684:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.860433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519158649075751:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.852058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439431:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.852084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519161855439431:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.852668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519161855439443:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-11T17:21:19.852811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519161855439444:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-11T17:21:19.854534Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519161855439434:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:19.858193Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519161855439431:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:19.863470Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519158649075751:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:19.831214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519161812338033:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.844859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519158669049682:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.832093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519161812338033:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:19.858972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049771:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.875371Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439443:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.875532Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439442:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.837742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519161812337967:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.859243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049770:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
2025-11-11T17:21:19.881758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519158649075743:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.878503Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519159292588439:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:19.841193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:19.861840Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519158669049764:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:19.843677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519161812337967:2049] [node 3] ICP22 created new session: [1:7571519161812338055:2049]
2025-11-11T17:21:19.875612Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439444:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.881850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:19.875649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519161855439426:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.885121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519159292588362:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.844961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519161812338055:2049] [node 3] ICS09 handshake done sender: [1:7571519161812338033:2110] self: [1:7571519161812338032:0] peer: [3:7571519158649075761:0] socket: 636 qp: -1
2025-11-11T17:21:19.862961Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519158669049762:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:19.883917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519158649075743:2107] [node 4] ICP22 created new session: [3:7571519158649075770:2107]
2025-11-11T17:21:19.875704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.884174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519158649075770:2107] [node 4] ICS09 handshake done sender: [3:7571519158649075751:2112] self: [3:7571519158649075749:0] peer: [4:7571519158669049769:0] socket: 643 qp: -1
2025-11-11T17:21:19.864062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519158669049682:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.845001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519161812338055:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:19.885195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:19.875757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519161855439426:2107] [node 4] ICP22 created new session: [2:7571519161855439449:2107]
2025-11-11T17:21:19.870085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:19.873188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519158669049682:2048] [node 1] ICP22 created new session: [4:7571519158669049773:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.845043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:19.885295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519158649075770:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:19.887492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:19.902228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519158649075772:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35934
2025-11-11T17:21:19.909794Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075772:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.926506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519158649075681:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.887620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519159292588362:2049] [node 2] ICP22 created new session: [5:7571519159292588445:2049]
2025-11-11T17:21:19.873261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519158669049773:2048] [node 1] ICS09 handshake done sender: [4:7571519158669049764:2118] self: [4:7571519158669049768:0] peer: [1:7571519161812338034:0] socket: 639 qp: -1
2025-11-11T17:21:19.873310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519158669049773:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:19.888011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519159292588445:2049] [node 2] ICS09 handshake done sender: [5:7571519159292588439:2115] self: [5:7571519159292588441:0] peer: [2:7571519161855439430:0] socket: 630 qp: -1
2025-11-11T17:21:19.845460Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338035:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.875800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519161855439449:2107] [node 4] ICS09 handshake done sender: [2:7571519161855439434:2112] self: [2:7571519161855439432:0] peer: [4:7571519158669049767:0] socket: 629 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.845483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519161812338035:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.873383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:19.875833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519161855439449:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:19.893146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519159292588445:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:19.873910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519158669049683:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.939835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519160607550015:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:19.940186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519160607550016:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:19.940301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519160607550018:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:19.893696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:19.875866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:19.940705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519158649075743:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(56790239ns) {HandleHandshakeStatus:329(56788143ns) {DropIncomingHandshake:441(21396ns) DropOutgoingHandshake:456(291ns) SwitchToState:178(427ns) ProcessPendingSessionEvents:495(37371248ns) {ForwardSessionEventToSession:612(24786265ns) ForwardSessionEventToSession:612(11495ns) ForwardSessionEventToSession:612(5976140ns) ForwardSessionEventToSession:612(3697ns) ForwardSessionEventToSession:612(2516997ns) ForwardSessionEventToSession:612(8557ns) ForwardSessionEventToSession:612(4119ns) ForwardSessionEventToSession:612(665037ns) ForwardSessionEventToSession:612(1175907ns) ForwardSessionEventToSession:612(6704ns)}}}
2025-11-11T17:21:19.854594Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519161812338035:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:19.946134Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519160607550015:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.946288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519160607549937:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.949218Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519160607550018:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.862050Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338039:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.876271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519161855439425:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.873962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:19.907811Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.965693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075755:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.965759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519158649075755:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.874007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519158669049683:2049] [node 2] ICP22 created new session: [4:7571519158669049775:2049]
2025-11-11T17:21:19.925996Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519159292588443:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.874045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519158669049775:2049] [node 2] ICS09 handshake done sender: [4:7571519158669049762:2117] self: [4:7571519158669049767:0] peer: [2:7571519161855439432:0] socket: 632 qp: -1
2025-11-11T17:21:19.959328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519160607549983:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.876297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:19.874075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519158669049775:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:19.862078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519161812338039:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.959450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519160607550020:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:19.926110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519159292588446:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:19.876333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519161855439425:2108] [node 5] ICP22 created new session: [2:7571519161855439451:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:20.016036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519158649075775:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35934
2025-11-11T17:21:20.016245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519162944043073:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35934
2025-11-11T17:21:19.864029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519161812338029:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.939216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519159292588361:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519159292588440:2116] held: no
2025-11-11T17:21:19.874978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:19.876372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519161855439451:2108] [node 5] ICS09 handshake done sender: [2:7571519161855439431:2111] self: [2:7571519161855439430:0] peer: [5:7571519159292588441:0] socket: 628 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.864123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:19.882136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519158669049772:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6154
2025-11-11T17:21:20.020569Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519158649075755:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.876404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519161855439451:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:20.049641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519160607550017:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:20.037960Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075775:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.050491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519158649075745:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.050546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:20.050606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519158649075745:2108] [node 5] ICP22 created new session: [3:7571519162944043074:2108]
2025-11-11T17:21:20.050655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519162944043074:2108] [node 5] ICS09 handshake done sender: [3:7571519158649075755:2115] self: [3:7571519158649075754:0] peer: [5:7571519159292588453:0] socket: 645 qp: -1
2025-11-11T17:21:20.050694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519162944043074:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:20.050729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:20.051243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519158649075745:2108] [node 5] ICP18 handshake (actor [3:7571519158649075775:2120]) is too old
2025-11-11T17:21:19.939569Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519159292588440:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:19.960053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519159292588361:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.960120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:19.960172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519159292588361:2048] [node 1] ICP22 created new session: [5:7571519159292588450:2048]
2025-11-11T17:21:19.960223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519159292588450:2048] [node 1] ICS09 handshake done sender: [5:7571519159292588440:2116] self: [5:7571519159292588442:0] peer: [1:7571519161812338038:0] socket: 650 qp: -1
2025-11-11T17:21:19.960256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519159292588450:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:19.960298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:19.960747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519159292588363:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.961247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519159292588449:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:19.982157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519159292588362:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(55581270ns) {HandleHandshakeStatus:329(55578564ns) {DropIncomingHandshake:441(1049ns) DropOutgoingHandshake:456(6219ns) SwitchToState:178(644ns) ProcessPendingSessionEvents:495(25735683ns) {ForwardSessionEventToSession:612(18166789ns) ForwardSessionEventToSession:612(1379209ns) ForwardSessionEventToSession:612(3585ns) ForwardSessionEventToSession:612(4560ns) ForwardSessionEventToSession:612(1291714ns) ForwardSessionEventToSession:612(3890ns) ForwardSessionEventToSession:612(2407ns) ForwardSessionEventToSession:612(2021129ns) ForwardSessionEventToSession:612(892534ns) ForwardSessionEventToSession:612(1648942ns)}}}
2025-11-11T17:21:19.985906Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519159292588449:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.986076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519159292588424:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:20.025865Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:20.046002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519159292588424:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519159292588449:2120] held: no
2025-11-11T17:21:20.046281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519163587555751:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:20.048557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519163587555752:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:20.048984Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519159292588449:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:20.050197Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519159292588443:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:20.052426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519159292588424:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.052476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:20.052523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519159292588424:2108] [node 4] ICP22 created new session: [5:7571519163587555753:2108]
2025-11-11T17:21:20.052567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519163587555753:2108] [node 4] ICS09 handshake done sender: [5:7571519159292588449:2120] self: [5:7571519159292588454:0] peer: [4:7571519158669049753:0] socket: 635 qp: -1
2025-11-11T17:21:20.052612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519163587555753:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:20.052651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:20.053027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519159292588363:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.053061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:20.053100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519159292588363:2050] [node 3] ICP22 created new session: [5:7571519163587555755:2050]
2025-11-11T17:21:20.053140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519163587555755:2050] [node 3] ICS09 handshake done sender: [5:7571519159292588443:2117] self: [5:7571519159292588453:0] peer: [3:7571519158649075754:0] socket: 656 qp: -1
2025-11-11T17:21:20.053176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519163587555755:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:20.053200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:19.864178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519161812338029:2107] [node 4] ICP22 created new session: [1:7571519161812338058:2107]
2025-11-11T17:21:19.892920Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519158669049765:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:19.876430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:20.058004Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.876827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519161855439445:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-11T17:21:19.883981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519161855439425:2108] [node 5] ICP18 handshake (actor [2:7571519161855439443:2117]) is too old
2025-11-11T17:21:19.887822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519161855439362:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:19.894223Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439445:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.938153Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439435:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.938194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519161855439435:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.939700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519161855439458:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-11T17:21:19.957977Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.973037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519161855439362:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519161855439444:2118] held: no
2025-11-11T17:21:19.977870Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519161855439444:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:19.985770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519161855439362:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.011058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:20.026636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519161855439362:2048] [node 1] ICP22 created new session: [2:7571519166150406756:2048]
2025-11-11T17:21:20.031881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519161855439435:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:20.037857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519166150406756:2048] [node 1] ICS09 handshake done sender: [2:7571519161855439444:2118] self: [2:7571519161855439456:0] peer: [1:7571519161812338036:0] socket: 646 qp: -1
2025-11-11T17:21:20.039325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519166150406756:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:20.042538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:20.064358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519161855439363:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.064454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:20.064510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519161855439363:2049] [node 3] ICP22 created new session: [2:7571519166150406758:2049]
2025-11-11T17:21:20.064558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519166150406758:2049] [node 3] ICS09 handshake done sender: [2:7571519161855439435:2113] self: [2:7571519161855439433:0] peer: [3:7571519158649075774:0] socket: 661 qp: -1
2025-11-11T17:21:20.064593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519166150406758:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:20.064921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:20.065502Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519161855439362:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(78986216ns) {HandleHandshakeStatus:329(78983092ns) {DropIncomingHandshake:441(903ns) DropOutgoingHandshake:456(531ns) SwitchToState:178(557ns) ProcessPendingSessionEvents:495(846ns)}}
2025-11-11T17:21:19.905766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519158669049684:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:19.864224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519161812338058:2107] [node 4] ICS09 handshake done sender: [1:7571519161812338035:2111] self: [1:7571519161812338034:0] peer: [4:7571519158669049768:0] socket: 637 qp: -1
2025-11-11T17:21:19.864255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519161812338058:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:20.061682Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519160607550016:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.905815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:19.864287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:19.887338Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519161812338039:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:19.893816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519161812338030:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.896135Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338037:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.896158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519161812338037:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.897134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:19.904567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519161812338030:2108] [node 5] ICP22 created new session: [1:7571519161812338062:2108]
2025-11-11T17:21:19.913870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519161812338062:2108] [node 5] ICS09 handshake done sender: [1:7571519161812338039:2113] self: [1:7571519161812338038:0] peer: [5:7571519159292588442:0] socket: 649 qp: -1
2025-11-11T17:21:19.913918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519161812338062:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:19.913975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:19.914423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519161812338030:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(20870184ns) {HandleHandshakeStatus:329(20406019ns) {DropIncomingHandshake:441(18042ns) DropOutgoingHandshake:456(418ns) SwitchToState:178(425ns) ProcessPendingSessionEvents:495(103518ns) {ForwardSessionEventToSession:612(56007ns) ForwardSessionEventToSession:612(2695ns) ForwardSessionEventToSession:612(3256ns) ForwardSessionEventToSession:612(5705ns) ForwardSessionEventToSession:612(2713ns) ForwardSessionEventToSession:612(5941ns) ForwardSessionEventToSession:612(3413ns) ForwardSessionEventToSession:612(4818ns) ForwardSessionEventToSession:612(6554ns) ForwardSessionEventToSession:612(2958ns)}}}
2025-11-11T17:21:19.921843Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519161812338037:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:19.933359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519161812337966:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:19.933447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:19.933504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519161812337966:2048] [node 2] ICP22 created new session: [1:7571519161812338065:2048]
2025-11-11T17:21:19.933561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519161812338065:2048] [node 2] ICS09 handshake done sender: [1:7571519161812338037:2112] self: [1:7571519161812338036:0] peer: [2:7571519161855439456:0] socket: 638 qp: -1
2025-11-11T17:21:19.934084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519161812338065:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:19.934129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:19.947447Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519161812338041:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:19.947474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519161812338041:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:19.948473Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519161812338041:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:19.993697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519161812338031:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.063057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519160607550015:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:20.067471Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519158649075772:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:19.993765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:20.009730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519161812338031:2109] [node 6] ICP22 created new session: [1:7571519161812338068:2109]
2025-11-11T17:21:20.009808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519161812338068:2109] [node 6] ICS09 handshake done sender: [1:7571519161812338041:2114] self: [1:7571519161812338040:0] peer: [6:7571519160607550019:0] socket: 633 qp: -1
2025-11-11T17:21:20.009869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519161812338068:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:20.009951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:20.010507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519161812338031:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(24417093ns) {HandleHandshakeStatus:329(24414817ns) {DropIncomingHandshake:441(1699ns) DropOutgoingHandshake:456(1146ns) SwitchToState:178(350ns) ProcessPendingSessionEvents:495(137878ns) {ForwardSessionEventToSession:612(78031ns) ForwardSessionEventToSession:612(3475ns) ForwardSessionEventToSession:612(7519ns) ForwardSessionEventToSession:612(3323ns) ForwardSessionEventToSession:612(9033ns) ForwardSessionEventToSession:612(3575ns) ForwardSessionEventToSession:612(2841ns) ForwardSessionEventToSession:612(11777ns) ForwardSessionEventToSession:612(3534ns) ForwardSessionEventToSession:612(3214ns)}}}
2025-11-11T17:21:19.905867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519158669049684:2050] [node 3] ICP22 created new session: [4:7571519158669049778:2050]
2025-11-11T17:21:20.070040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519160607549983:2094] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519160607550018:2118] held: no
2025-11-11T17:21:20.070217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519160607550022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:20.070327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519160607549937:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.070405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:20.070442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519160607549937:2048] [node 1] ICP22 created new session: [6:7571519164902517320:2048]
2025-11-11T17:21:20.070489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519164902517320:2048] [node 1] ICS09 handshake done sender: [6:7571519160607550015:2115] self: [6:7571519160607550019:0] peer: [1:7571519161812338040:0] socket: 660 qp: -1
2025-11-11T17:21:20.070683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519160607550017:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.077777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519164902517320:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:20.077832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:20.078479Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519160607550022:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.078630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519164902517319:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:20.067738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519158649075681:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.067782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:20.068745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519158649075681:2049] [node 2] ICP22 created new session: [3:7571519162944043076:2049]
2025-11-11T17:21:20.068793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519162944043076:2049] [node 2] ICS09 handshake done sender: [3:7571519158649075772:2119] self: [3:7571519158649075774:0] peer: [2:7571519161855439433:0] socket: 623 qp: -1
2025-11-11T17:21:20.069653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519162944043076:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:20.069702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:19.905912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519158669049778:2050] [node 3] ICS09 handshake done sender: [4:7571519158669049765:2119] self: [4:7571519158669049769:0] peer: [3:7571519158649075749:0] socket: 644 qp: -1
2025-11-11T17:21:19.905948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519158669049778:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:19.905983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:19.906524Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:19.937704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519158669049684:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:20.002026Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158669049754:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.002075Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519158669049754:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:20.005527Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519158669049756:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.005577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519158669049756:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:20.013700Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519158669049754:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:20.021919Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519158669049756:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:20.022382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519158669049745:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.022448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:20.022497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519158669049745:2108] [node 5] ICP22 created new session: [4:7571519162964017079:2108]
2025-11-11T17:21:20.022540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519162964017079:2108] [node 5] ICS09 handshake done sender: [4:7571519158669049754:2113] self: [4:7571519158669049753:0] peer: [5:7571519159292588454:0] socket: 614 qp: -1
2025-11-11T17:21:20.022577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519162964017079:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:20.022611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:20.022972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519158669049746:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.023012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:20.023056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519158669049746:2109] [node 6] ICP22 created new session: [4:7571519162964017081:2109]
2025-11-11T17:21:20.023090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519162964017081:2109] [node 6] ICS09 handshake done sender: [4:7571519158669049756:2114] self: [4:7571519158669049755:0] peer: [6:7571519160607550021:0] socket: 618 qp: -1
2025-11-11T17:21:20.023114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519162964017081:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:20.023142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:20.089889Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519161855439437:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.089935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519161855439437:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:20.091027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519163587555757:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-11T17:21:20.091501Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519161855439437:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:20.091827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519161855439427:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.091922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:20.091976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519161855439427:2109] [node 6] ICP22 created new session: [2:7571519166150406761:2109]
2025-11-11T17:21:20.092020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519166150406761:2109] [node 6] ICS09 handshake done sender: [2:7571519161855439437:2114] self: [2:7571519161855439436:0] peer: [6:7571519164902517322:0] socket: 631 qp: -1
2025-11-11T17:21:20.092055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519166150406761:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:20.092095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:20.094405Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519158649075766:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.094440Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519158649075766:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:20.087638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519160607549938:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:20.088193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519160607549939:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:20.088265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519160607549992:2091] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:20.088476Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519160607550018:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:20.098602Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519158649075766:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:20.094450Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519159292588433:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.094488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519159292588433:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:20.098505Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519159292588433:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:20.098748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519159292588425:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.098783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:20.098816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519159292588425:2109] [node 6] ICP22 created new session: [5:7571519163587555759:2109]
2025-11-11T17:21:20.098865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519163587555759:2109] [node 6] ICS09 handshake done sender: [5:7571519159292588433:2113] self: [5:7571519159292588432:0] peer: [6:7571519164902517325:0] socket: 667 qp: -1
2025-11-11T17:21:20.098891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519163587555759:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:20.098933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:20.102026Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:20.103433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519160607549983:2094] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.103480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:20.103591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519160607549983:2094] [node 4] ICP22 created new session: [6:7571519164902517329:2094]
2025-11-11T17:21:20.103640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519164902517329:2094] [node 4] ICS09 handshake done sender: [6:7571519160607550018:2118] self: [6:7571519160607550021:0] peer: [4:7571519158669049755:0] socket: 664 qp: -1
2025-11-11T17:21:20.103676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519164902517329:2094] [node 4] ICS10 traffic start
2025-11-11T17:21:20.103743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:20.104477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519160607549938:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519160607550016:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:20.105022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519164902517326:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:20.105361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519164902517327:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:20.106100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519158649075744:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:20.106198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:20.106257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519158649075744:2109] [node 6] ICP22 created new session: [3:7571519162944043079:2109]
2025-11-11T17:21:20.106296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519162944043079:2109] [node 6] ICS09 handshake done sender: [3:7571519158649075766:2118] self: [3:7571519158649075765:0] peer: [6:7571519164902517324:0] socket: 662 qp: -1
2025-11-11T17:21:20.106328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519162944043079:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:20.106359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:20.105752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519164902517328:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4156
2025-11-11T17:21:20.108040Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:20.109191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519160607550016:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:20.110841Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519160607550017:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:20.111217Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519160607550022:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:20.111430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519160607549939:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519160607550017:2117] held: no
2025-11-11T17:21:20.111527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519160607549939:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.111586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:20.111620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519160607549939:2050] [node 3] ICP22 created new session: [6:7571519164902517331:2050]
2025-11-11T17:21:20.111660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519164902517331:2050] [node 3] ICS09 handshake done sender: [6:7571519160607550017:2117] self: [6:7571519164902517324:0] peer: [3:7571519158649075765:0] socket: 619 qp: -1
2025-11-11T17:21:20.111688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519164902517331:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:20.111736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:20.112121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519160607549938:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.112140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:20.112170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519160607549938:2049] [node 2] ICP22 created new session: [6:7571519164902517333:2049]
2025-11-11T17:21:20.112193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519164902517333:2049] [node 2] ICS09 handshake done sender: [6:7571519160607550016:2116] self: [6:7571519164902517322:0] peer: [2:7571519161855439436:0] socket: 634 qp: -1
2025-11-11T17:21:20.112210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519164902517333:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:20.112258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:20.112756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519160607549992:2091] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:20.112806Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519163587555757:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:20.112788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:20.112952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519160607549992:2091] [node 5] ICP22 created new session: [6:7571519164902517335:2091]
2025-11-11T17:21:20.112982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519164902517335:2091] [node 5] ICS09 handshake done sender: [6:7571519160607550022:2120] self: [6:7571519164902517325:0] peer: [5:7571519159292588432:0] socket: 669 qp: -1
2025-11-11T17:21:20.113315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519164902517335:2091] [node 5] ICS10 traffic start
2025-11-11T17:21:20.113353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:20.113142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571519159292588425:2109] [node 6] ICP18 handshake (actor [5:7571519163587555757:2123]) is too old
2025-11-11T17:21:20.181768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:20.181985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519166150406756:2048] [node 1] ICS01 socket: 646 reason# EndOfStream
2025-11-11T17:21:20.182034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519161855439362:2048] [node 1] ICP30 unregister session Session# [2:7571519166150406756:2048] VirtualId# [2:7571519161855439456:0]
2025-11-11T17:21:20.182068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519166150406756:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.182157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:20.182199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:20.183424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:20.183650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519158649075763:2048] [node 1] ICS01 socket: 640 reason# EndOfStream
2025-11-11T17:21:20.183694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519158649075680:2048] [node 1] ICP30 unregister session Session# [3:7571519158649075763:2048] VirtualId# [3:7571519158649075761:0]
2025-11-11T17:21:20.183728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519158649075763:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.183810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:20.183850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:20.193735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:20.198172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519159292588450:2048] [node 1] ICS01 socket: 650 reason# EndOfStream
2025-11-11T17:21:20.198221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519159292588361:2048] [node 1] ICP30 unregister session Session# [5:7571519159292588450:2048] VirtualId# [5:7571519159292588442:0]
2025-11-11T17:21:20.198261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519159292588450:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.198382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:20.198432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:20.193970Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:20.197762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:20.204980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519164902517320:2048] [node 1] ICS01 socket: 660 reason# EndOfStream
2025-11-11T17:21:20.213996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519158669049773:2048] [node 1] ICS01 socket: 639 reason# EndOfStream
2025-11-11T17:21:20.208870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519160607549937:2048] [node 1] ICP30 unregister session Session# [6:7571519164902517320:2048] VirtualId# [6:7571519160607550019:0]
2025-11-11T17:21:20.208936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519164902517320:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.210550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:20.210653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:20.225748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519158669049682:2048] [node 1] ICP30 unregister session Session# [4:7571519158669049773:2048] VirtualId# [4:7571519158669049768:0]
2025-11-11T17:21:20.225813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519158669049773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.225924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:20.225976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:20.317282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:20.317493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519162944043076:2049] [node 2] ICS01 socket: 623 reason# EndOfStream
2025-11-11T17:21:20.317537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519158649075681:2049] [node 2] ICP30 unregister session Session# [3:7571519162944043076:2049] VirtualId# [3:7571519158649075774:0]
2025-11-11T17:21:20.317575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519162944043076:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.317675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:20.317726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:20.325814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:20.326062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519158669049775:2049] [node 2] ICS07 socket disconnect 632 reason# EndOfStream
2025-11-11T17:21:20.326095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519158669049775:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.326183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:20.326228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519158669049775:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:20.327676Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address
2025-11-11T17:21:20.327926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519158669049683:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:20.327976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519158669049775:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:20.328004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519158669049683:2049] [node 2] ICP30 unregister session Session# [4:7571519158669049775:2049] VirtualId# [4:7571519158669049767:0]
2025-11-11T17:21:20.328048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:20.328080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519158669049683:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:20.328095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:20.326722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:20.326931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519164902517333:2049] [node 2] ICS01 socket: 634 reason# EndOfStream
2025-11-11T17:21:20.326969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519160607549938:2049] [node 2] ICP30 unregister session Session# [6:7571519164902517333:2049] VirtualId# [6:7571519164902517322:0]
2025-11-11T17:21:20.327002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519164902517333:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.327063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:20.327107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:20.329709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:20.329968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519159292588445:2049] [node 2] ICS07 socket disconnect 630 reason# EndOfStream
2025-11-11T17:21:20.329997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519159292588445:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.330077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:20.330121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519159292588445:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:20.335249Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address
2025-11-11T17:21:20.335567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519159292588362:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:20.335621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519159292588445:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:20.335659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519159292588362:2049] [node 2] ICP30 unregister session Session# [5:7571519159292588445:2049] VirtualId# [5:7571519159292588441:0]
2025-11-11T17:21:20.335706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:20.335738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519159292588362:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:20.335754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:20.339314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519158669049683:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:20.349698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519159292588362:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:20.413809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:20.414029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519163587555755:2050] [node 3] ICS01 socket: 656 reason# EndOfStream
2025-11-11T17:21:20.414068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519159292588363:2050] [node 3] ICP30 unregister session Session# [5:7571519163587555755:2050] VirtualId# [5:7571519159292588453:0]
2025-11-11T17:21:20.414107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519163587555755:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.414185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:20.414230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:20.416961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:20.417231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519158669049778:2050] [node 3] ICS01 socket: 644 reason# EndOfStream
2025-11-11T17:21:20.414318Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:20.417296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519158669049684:2050] [node 3] ICP30 unregister session Session# [4:7571519158669049778:2050] VirtualId# [4:7571519158669049769:0]
2025-11-11T17:21:20.414476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519164902517331:2050] [node 3] ICS01 socket: 619 reason# EndOfStream
2025-11-11T17:21:20.417344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519158669049778:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.414503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519160607549939:2050] [node 3] ICP30 unregister session Session# [6:7571519164902517331:2050] VirtualId# [6:7571519164902517324:0]
2025-11-11T17:21:20.417453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:20.417502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:20.414528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519164902517331:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.414569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:20.414600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:20.481521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:20.481786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519163587555753:2108] [node 4] ICS01 socket: 635 reason# EndOfStream
2025-11-11T17:21:20.481833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519159292588424:2108] [node 4] ICP30 unregister session Session# [5:7571519163587555753:2108] VirtualId# [5:7571519159292588454:0]
2025-11-11T17:21:20.481872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519163587555753:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.481962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:20.482006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:20.482541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:20.482729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519164902517329:2094] [node 4] ICS01 socket: 664 reason# EndOfStream
2025-11-11T17:21:20.482759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519160607549983:2094] [node 4] ICP30 unregister session Session# [6:7571519164902517329:2094] VirtualId# [6:7571519160607550021:0]
2025-11-11T17:21:20.482789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519164902517329:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.482849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:20.482884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:20.569778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:20.570013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519164902517335:2091] [node 5] ICS07 socket disconnect 669 reason# EndOfStream
2025-11-11T17:21:20.570045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519164902517335:2091] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:20.570169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:20.570216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519164902517335:2091] [node 5] ICS15 start handshake
2025-11-11T17:21:20.571114Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26245) Couldn't connect to any resolved address
2025-11-11T17:21:20.571338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519160607549992:2091] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26245) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:20.571375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519164902517335:2091] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:20.571418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519160607549992:2091] [node 5] ICP30 unregister session Session# [6:7571519164902517335:2091] VirtualId# [6:7571519164902517325:0]
2025-11-11T17:21:20.571468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:21:20.571497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519160607549992:2091] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26245) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:20.571511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26245) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:20.670203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519163755872269:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:20.670559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519163755872270:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:20.690976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519165733636752:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:20.691018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519165733636753:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:20.716126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519165534061531:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:20.716180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519165534061532:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:20.740519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519165440915818:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:20.740535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519165440915819:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:20.740585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519165440915820:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:20.785910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519164061103493:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:20.785968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519164061103494:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:20.785999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519164061103495:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:20.806188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519162648215729:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:20.806252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519162648215731:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:20.806565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519162648215730:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:20.958509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519162648215792:2106] [node 5] ICP01 ready to work
2025-11-11T17:21:20.980958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519163755872332:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:21.013961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519165733636816:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:21.037862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519168356070851:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:21.037909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519169829028891:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:21.026560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519163755872333:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:21.038215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519169829028892:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:21.033777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519165733636815:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:21.047135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519169735883177:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:21.038472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519169829028893:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:21.047054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519170028604113:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:21.048540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519168356070852:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:21.014202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519168050839630:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:21.052905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519165534061531:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(85026667ns) {Configure:104(85024100ns) {ProcessConfigured:135(42753ns) {StartInitialHandshake:157(40957ns) {DropHandshakes:469(2551ns) {DropIncomingHandshake:441(349ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:476(855ns) {SwitchToState:178(169ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.052310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519169735883178:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:21.056855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519166943183091:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:21.058470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519162648215792:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(77205494ns) {Configure:104(77197000ns) {ProcessConfigured:135(47120ns) {StartInitialHandshake:157(45105ns) {DropHandshakes:469(1832ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(380ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:476(1630ns) {SwitchToState:178(605ns)}}}}}
2025-11-11T17:21:21.060421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183096:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.062077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519165534061532:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(89992611ns) {Configure:104(89986578ns) {ProcessConfigured:135(4708825ns) {StartInitialHandshake:157(4705918ns) {DropHandshakes:469(2106ns) {DropIncomingHandshake:441(476ns) DropOutgoingHandshake:456(487ns)} GenerateSessionVirtualId:988(513ns) PrepareNewSessionHandshake:476(2958ns) {SwitchToState:178(1392ns)}}}}}
2025-11-11T17:21:21.062088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519165440915819:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(83874260ns) {Configure:104(83865499ns) {ProcessConfigured:135(5876483ns) {StartInitialHandshake:157(5873245ns) {DropHandshakes:469(2540ns) {DropIncomingHandshake:441(589ns) DropOutgoingHandshake:456(508ns)} GenerateSessionVirtualId:988(987ns) PrepareNewSessionHandshake:476(4091ns) {SwitchToState:178(1845ns)}}}}}
2025-11-11T17:21:21.062733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519165440915818:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(86010103ns) {Configure:104(86006258ns) {ProcessConfigured:135(52025ns) {StartInitialHandshake:157(47173ns) {DropHandshakes:469(1888ns) {DropIncomingHandshake:441(561ns) DropOutgoingHandshake:456(387ns)} GenerateSessionVirtualId:988(708ns) PrepareNewSessionHandshake:476(956ns) {SwitchToState:178(442ns)}}}}}
2025-11-11T17:21:21.066659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519162648215729:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(76643950ns) {Configure:104(76638415ns) {ProcessConfigured:135(2188763ns) {StartInitialHandshake:157(2186481ns) {DropHandshakes:469(1870ns) {DropIncomingHandshake:441(533ns) DropOutgoingHandshake:456(499ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:476(3133ns) {SwitchToState:178(960ns)}}}}}
2025-11-11T17:21:21.076602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519163755872269:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(133367766ns) {Configure:104(133362048ns) {ProcessConfigured:135(2870317ns) {StartInitialHandshake:157(2868000ns) {DropHandshakes:469(3217ns) {DropIncomingHandshake:441(546ns) DropOutgoingHandshake:456(923ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:476(3050ns) {SwitchToState:178(1567ns)}}}}}
2025-11-11T17:21:21.075194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519165733636816:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22736662ns) {Configure:104(22732544ns) {ProcessConfigured:135(70128ns) {StartInitialHandshake:157(66843ns) {DropHandshakes:469(3383ns) {DropIncomingHandshake:441(681ns) DropOutgoingHandshake:456(620ns)} GenerateSessionVirtualId:988(1145ns) PrepareNewSessionHandshake:476(2106ns) {SwitchToState:178(911ns)}}}}}
2025-11-11T17:21:21.090554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519166943183091:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28227587ns) {Configure:104(28223129ns) {ProcessConfigured:135(68156ns) {StartInitialHandshake:157(65800ns) {DropHandshakes:469(2835ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(412ns)} GenerateSessionVirtualId:988(1477ns) PrepareNewSessionHandshake:476(1370ns) {SwitchToState:178(438ns)}}}}}
2025-11-11T17:21:21.090729Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519166943183096:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.092337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519162648215731:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.092766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028899:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.078465Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519164061103493:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(29643322ns) {Configure:104(29638526ns) {ProcessConfigured:135(528033ns) {StartInitialHandshake:157(525750ns) {DropHandshakes:469(1520ns) {DropIncomingHandshake:441(357ns) DropOutgoingHandshake:456(188ns)} GenerateSessionVirtualId:988(479ns) PrepareNewSessionHandshake:476(1743ns) {SwitchToState:178(702ns)}}}}}
2025-11-11T17:21:21.079744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519164061103495:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30162384ns) {Configure:104(30153286ns) {ProcessConfigured:135(786188ns) {StartInitialHandshake:157(784090ns) {DropHandshakes:469(978ns) {DropIncomingHandshake:441(256ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:476(2910ns) {SwitchToState:178(927ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.092803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519169829028899:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.104593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519165440915820:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19119451ns) {Configure:104(19111408ns) {ProcessConfigured:135(2331368ns) {StartInitialHandshake:157(2327771ns) {DropHandshakes:469(1110ns) {DropIncomingHandshake:441(271ns) DropOutgoingHandshake:456(190ns)} GenerateSessionVirtualId:988(331ns) PrepareNewSessionHandshake:476(1965ns) {SwitchToState:178(1104ns)}}}}}
2025-11-11T17:21:21.097738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519164061103494:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(41005140ns) {Configure:104(40997119ns) {ProcessConfigured:135(3614332ns) {StartInitialHandshake:157(3611297ns) {DropHandshakes:469(2920ns) {DropIncomingHandshake:441(755ns) DropOutgoingHandshake:456(621ns)} GenerateSessionVirtualId:988(1163ns) PrepareNewSessionHandshake:476(3371ns) {SwitchToState:178(1670ns)}}}}}
2025-11-11T17:21:21.108592Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519163755872332:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(68572076ns) {Configure:104(68559439ns) {ProcessConfigured:135(9088600ns) {StartInitialHandshake:157(9074006ns) {DropHandshakes:469(3041ns) {DropIncomingHandshake:441(777ns) DropOutgoingHandshake:456(632ns)} GenerateSessionVirtualId:988(873ns) PrepareNewSessionHandshake:476(4674ns) {SwitchToState:178(2179ns)}}}}}
2025-11-11T17:21:21.108093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519165733636753:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(49035457ns) {Configure:104(49029520ns) {ProcessConfigured:135(11561852ns) {StartInitialHandshake:157(11558299ns) {DropHandshakes:469(2265ns) {DropIncomingHandshake:441(554ns) DropOutgoingHandshake:456(506ns)} GenerateSessionVirtualId:988(724ns) PrepareNewSessionHandshake:476(2617ns) {SwitchToState:178(989ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.108185Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519169829028899:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-11T17:21:21.113974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519168050839630:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(58855321ns) {Configure:104(58848813ns) {ProcessConfigured:135(2834113ns) {StartInitialHandshake:157(2830916ns) {DropHandshakes:469(2494ns) {DropIncomingHandshake:441(804ns) DropOutgoingHandshake:456(404ns)} GenerateSessionVirtualId:988(874ns) PrepareNewSessionHandshake:476(4211ns) {SwitchToState:178(1787ns)}}}}}
::ffff:127.1.0.0
2025-11-11T17:21:21.114067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519165733636815:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(33296298ns) {Configure:104(33290959ns) {ProcessConfigured:135(1818332ns) {StartInitialHandshake:157(1815351ns) {DropHandshakes:469(1706ns) {DropIncomingHandshake:441(520ns) DropOutgoingHandshake:456(177ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:476(386974ns) {SwitchToState:178(1802ns)}}}}}
2025-11-11T17:21:21.118002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519169735883177:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(45471988ns) {Configure:104(45468139ns) {ProcessConfigured:135(4039869ns) {StartInitialHandshake:157(4037826ns) {DropHandshakes:469(2804ns) {DropIncomingHandshake:441(672ns) DropOutgoingHandshake:456(498ns)} GenerateSessionVirtualId:988(822ns) PrepareNewSessionHandshake:476(2263ns) {SwitchToState:178(1213ns)}}}}}
2025-11-11T17:21:21.109203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519169829028907:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14127
2025-11-11T17:21:21.122533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519170028604123:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23825
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.131002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519168356070851:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(44836042ns) {Configure:104(44830465ns) {ProcessConfigured:135(9390409ns) {StartInitialHandshake:157(9386571ns) {DropHandshakes:469(2659ns) {DropIncomingHandshake:441(542ns) DropOutgoingHandshake:456(363ns)} GenerateSessionVirtualId:988(922ns) PrepareNewSessionHandshake:476(5880ns) {SwitchToState:178(2866ns)}}}}}
2025-11-11T17:21:21.128020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519169829028893:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.130980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183107:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.131076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519163755872333:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(46424873ns) {Configure:104(46419030ns) {ProcessConfigured:135(14982115ns) {StartInitialHandshake:157(14979907ns) {DropHandshakes:469(5001430ns) {DropIncomingHandshake:441(1594ns) DropOutgoingHandshake:456(725ns)} GenerateSessionVirtualId:988(881ns) PrepareNewSessionHandshake:476(2811ns) {SwitchToState:178(978ns)}}}}}
2025-11-11T17:21:21.128073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:21.128510Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028907:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.132532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070860:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.127069Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604123:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.136225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519163755872270:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27426334ns) {Configure:104(27422418ns) {ProcessConfigured:135(61674ns) {StartInitialHandshake:157(59598ns) {DropHandshakes:469(2476ns) {DropIncomingHandshake:441(582ns) DropOutgoingHandshake:456(678ns)} GenerateSessionVirtualId:988(1065ns) PrepareNewSessionHandshake:476(1544ns) {SwitchToState:178(614ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.133043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070864:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.129290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519169829028893:2110] [node 6] ICP22 created new session: [3:7571519169829028909:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.151297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519168050839644:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3357
2025-11-11T17:21:21.135496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519170028604113:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(75078432ns) {Configure:104(75072939ns) {ProcessConfigured:135(5016987ns) {StartInitialHandshake:157(5014442ns) {DropHandshakes:469(3356376ns) {DropIncomingHandshake:441(741ns) DropOutgoingHandshake:456(1215ns)} GenerateSessionVirtualId:988(1657ns) PrepareNewSessionHandshake:476(3117ns) {SwitchToState:178(1375ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.140609Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519168356070860:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.153202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519168050839646:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3357
2025-11-11T17:21:21.135560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519169829028909:2110] [node 6] ICS09 handshake done sender: [3:7571519169829028899:2113] self: [3:7571519169829028898:0] peer: [6:7571519166943183106:0] socket: 620 qp: -1
2025-11-11T17:21:21.144719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519170028604129:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23825
2025-11-11T17:21:21.144787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519170028604130:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23825
2025-11-11T17:21:21.153409Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604130:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.153776Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.154294Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519169735883178:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(53805001ns) {Configure:104(53792097ns) {ProcessConfigured:135(8491633ns) {StartInitialHandshake:157(8488429ns) {DropHandshakes:469(3746ns) {DropIncomingHandshake:441(767ns) DropOutgoingHandshake:456(986ns)} GenerateSessionVirtualId:988(926ns) PrepareNewSessionHandshake:476(6700ns) {SwitchToState:178(1654ns)}}}}}
2025-11-11T17:21:21.141135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519169829028909:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:21.153606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519164061103495:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.154254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519165733636752:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.163363Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519166943183096:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:21.154018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519168050839645:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3357
2025-11-11T17:21:21.164579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183108:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.141204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:21.165024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519170028604132:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23825
2025-11-11T17:21:21.163781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883183:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.183233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.163018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839644:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.173970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519169735883183:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.175986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519162648215731:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519166943183096:2114] held: no
2025-11-11T17:21:21.141556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519169829028893:2110] [node 6] ICP18 handshake (actor [3:7571519169829028907:2116]) is too old
2025-11-11T17:21:21.177876Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519166943183108:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.179338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519162648215731:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.183161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519162648215730:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.184508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:21.188066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519162648215731:2050] [node 3] ICP22 created new session: [6:7571519166943183110:2050]
2025-11-11T17:21:21.194245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519166943183110:2050] [node 3] ICS09 handshake done sender: [6:7571519166943183096:2114] self: [6:7571519166943183106:0] peer: [3:7571519169829028898:0] socket: 621 qp: -1
2025-11-11T17:21:21.169982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519169829028911:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14127
2025-11-11T17:21:21.178480Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839632:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.174121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883190:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.196824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519166943183110:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:21.178528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519168050839632:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.196613Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604122:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.206199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519168356070852:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(60995449ns) {Configure:104(60986605ns) {ProcessConfigured:135(5267379ns) {StartInitialHandshake:157(5264274ns) {DropHandshakes:469(515494ns) {DropIncomingHandshake:441(1491ns) DropOutgoingHandshake:456(349ns)} GenerateSessionVirtualId:988(992ns) PrepareNewSessionHandshake:476(2206ns) {SwitchToState:178(672ns)}}}}}
2025-11-11T17:21:21.178188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519165440915820:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.196882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:21.170545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519169829028913:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14127
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.196669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519170028604122:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.178645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839645:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.208567Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519168356070861:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.181896Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028895:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.183111Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519169735883190:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.191512Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519168050839632:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.209054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604132:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.209848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519168356070864:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.224148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519162648215731:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(38273029ns) {HandleHandshakeStatus:329(37667795ns) {DropIncomingHandshake:441(411ns) DropOutgoingHandshake:456(486ns) SwitchToState:178(414ns) ProcessPendingSessionEvents:495(1001ns)}}
2025-11-11T17:21:21.181926Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519169829028895:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.187174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519165440915818:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.191083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883196:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
2025-11-11T17:21:21.205879Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519169735883183:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:21.211934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519165440915820:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.212005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:21.214099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519165440915820:2050] [node 3] ICP22 created new session: [4:7571519169735883197:2050]
2025-11-11T17:21:21.219893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519169735883197:2050] [node 3] ICS09 handshake done sender: [4:7571519169735883183:2112] self: [4:7571519169735883194:0] peer: [3:7571519169829028894:0] socket: 622 qp: -1
2025-11-11T17:21:21.224161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519169735883197:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:21.224207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.211660Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519170028604122:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:21.248753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883199:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
2025-11-11T17:21:21.232647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183113:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.194515Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519169829028895:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:21.205929Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839634:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.220328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070868:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.253157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183114:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.205970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519168050839634:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.225077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519168356070868:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.267429Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519166943183108:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:21.195135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519169829028891:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.258703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519165440915820:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(47414012ns) {HandleHandshakeStatus:329(46537633ns) {DropIncomingHandshake:441(567ns) DropOutgoingHandshake:456(10601ns) SwitchToState:178(871ns) ProcessPendingSessionEvents:495(9415284ns) {ForwardSessionEventToSession:612(9358968ns) ForwardSessionEventToSession:612(6484ns) ForwardSessionEventToSession:612(3354ns) ForwardSessionEventToSession:612(10849ns) ForwardSessionEventToSession:612(3384ns) ForwardSessionEventToSession:612(2614ns) ForwardSessionEventToSession:612(8482ns) ForwardSessionEventToSession:612(2937ns) ForwardSessionEventToSession:612(5290ns) ForwardSessionEventToSession:612(2433ns)}}}
2025-11-11T17:21:21.267330Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604129:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.225677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519164061103494:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.195202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:21.217732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519168050839648:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3357
2025-11-11T17:21:21.267968Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604118:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.264039Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.269763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519162648215730:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.267700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519165440915818:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519169735883190:2116] held: no
2025-11-11T17:21:21.269115Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519169735883190:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:21.276602Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519169735883187:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.276640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519169735883187:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.221199Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519168050839634:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:21.195252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519169829028891:2107] [node 4] ICP22 created new session: [3:7571519169829028915:2107]
2025-11-11T17:21:21.246781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519164061103493:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.269812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:21.268006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519170028604118:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.224276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519163755872269:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.224341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:21.225581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519163755872269:2048] [node 2] ICP22 created new session: [1:7571519168050839654:2048]
2025-11-11T17:21:21.229938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519168050839652:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3357
2025-11-11T17:21:21.282095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519165440915818:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.248690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070873:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.195298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519169829028915:2107] [node 4] ICS09 handshake done sender: [3:7571519169829028895:2111] self: [3:7571519169829028894:0] peer: [4:7571519169735883194:0] socket: 618 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-11T17:21:21.232836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519168050839654:2048] [node 2] ICS09 handshake done sender: [1:7571519168050839632:2110] self: [1:7571519168050839631:0] peer: [2:7571519170028604131:0] socket: 644 qp: -1
::ffff:127.1.0.0
2025-11-11T17:21:21.242016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839648:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.268964Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.272778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519166943183114:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.282139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:21.254449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070872:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.195331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519169829028915:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:21.273577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519162648215730:2049] [node 2] ICP22 created new session: [6:7571519166943183115:2049]
2025-11-11T17:21:21.264357Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519168356070873:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.282180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519165440915818:2048] [node 1] ICP22 created new session: [4:7571519169735883201:2048]
2025-11-11T17:21:21.242076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519163755872332:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.195368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:21.269528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519165733636815:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.278539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519166943183115:2049] [node 2] ICS09 handshake done sender: [6:7571519166943183108:2119] self: [6:7571519166943183109:0] peer: [2:7571519170028604121:0] socket: 635 qp: -1
2025-11-11T17:21:21.282741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519169735883201:2048] [node 1] ICS09 handshake done sender: [4:7571519169735883190:2116] self: [4:7571519169735883195:0] peer: [1:7571519168050839633:0] socket: 627 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.269603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:21.242142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:21.268675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519168356070851:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.200104Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028897:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.284648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519169735883201:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:21.278594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519166943183115:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:21.281702Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.282349Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519168356070860:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:21.283470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519164061103495:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519168356070860:2113] held: no
2025-11-11T17:21:21.285011Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.289137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519164061103493:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519168356070868:2117] held: no
2025-11-11T17:21:21.289926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070878:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.290006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519164061103495:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.292192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:21.292621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519164061103495:2050] [node 3] ICP22 created new session: [5:7571519168356070883:2050]
2025-11-11T17:21:21.293100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070879:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.293427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519168356070880:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48200
2025-11-11T17:21:21.297077Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519168356070868:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:21.297425Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519168356070873:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:21.297656Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519168356070864:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:21.269645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519165733636815:2109] [node 6] ICP22 created new session: [2:7571519170028604137:2109]
2025-11-11T17:21:21.200135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519169829028897:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.297878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519164061103493:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.242200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519163755872332:2107] [node 4] ICP22 created new session: [1:7571519168050839655:2107]
2025-11-11T17:21:21.278652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:21.270417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519165733636752:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519170028604130:2117] held: no
2025-11-11T17:21:21.284685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:21.278771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519162648215729:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.242252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519168050839655:2107] [node 4] ICS09 handshake done sender: [1:7571519168050839634:2111] self: [1:7571519168050839633:0] peer: [4:7571519169735883195:0] socket: 645 qp: -1
2025-11-11T17:21:21.297935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:21.203832Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519169829028897:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:21.288111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883203:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
2025-11-11T17:21:21.242291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519168050839655:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:21.242329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:21.242803Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839646:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.243180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519163755872332:2107] [node 4] ICP18 handshake (actor [1:7571519168050839648:2118]) is too old
2025-11-11T17:21:21.245170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519168050839654:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:21.254546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.257532Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839652:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.257651Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839638:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.257674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519168050839638:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.268960Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519168050839638:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:21.280714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519163755872333:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.280775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:21.282649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519163755872333:2108] [node 5] ICP22 created new session: [1:7571519168050839662:2108]
2025-11-11T17:21:21.284528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519168050839662:2108] [node 5] ICS09 handshake done sender: [1:7571519168050839638:2113] self: [1:7571519168050839637:0] peer: [5:7571519168356070876:0] socket: 646 qp: -1
2025-11-11T17:21:21.290449Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519169735883187:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:21.294529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519169735883177:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.284704Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839636:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.294559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:21.305672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183119:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.294618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519169735883177:2108] [node 5] ICP22 created new session: [4:7571519169735883207:2108]
2025-11-11T17:21:21.284732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519168050839636:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.305753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183120:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.284775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519168050839662:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:21.294665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519169735883207:2108] [node 5] ICS09 handshake done sender: [4:7571519169735883187:2114] self: [4:7571519169735883186:0] peer: [5:7571519168356070877:0] socket: 667 qp: -1
2025-11-11T17:21:21.285084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:21.294707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519169735883207:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:21.279028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519170028604137:2109] [node 6] ICS09 handshake done sender: [2:7571519170028604122:2113] self: [2:7571519170028604121:0] peer: [6:7571519166943183109:0] socket: 634 qp: -1
2025-11-11T17:21:21.294739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:21.290892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519163755872269:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(56428762ns) {HandleHandshakeStatus:329(56425640ns) {DropIncomingHandshake:441(16009ns) DropOutgoingHandshake:456(569ns) SwitchToState:178(710ns) ProcessPendingSessionEvents:495(10326680ns) {ForwardSessionEventToSession:612(2687530ns) ForwardSessionEventToSession:612(12581ns) ForwardSessionEventToSession:612(1468211ns) ForwardSessionEventToSession:612(2671ns) ForwardSessionEventToSession:612(740834ns) ForwardSessionEventToSession:612(3703ns) ForwardSessionEventToSession:612(2918913ns) ForwardSessionEventToSession:612(762775ns) ForwardSessionEventToSession:612(4421ns) ForwardSessionEventToSession:612(2784ns)}}}
2025-11-11T17:21:21.281839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519170028604137:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:21.295369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883206:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
2025-11-11T17:21:21.282019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:21.292509Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519168050839640:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.285436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519170028604136:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23825
2025-11-11T17:21:21.292537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519168050839640:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.286331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519170028604133:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23825
2025-11-11T17:21:21.292665Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519168050839636:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:21.290183Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604136:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.293185Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519170028604118:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:21.293865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519168050839630:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.293659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519165733636815:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(21598805ns) {HandleHandshakeStatus:329(21292168ns) {DropIncomingHandshake:441(11290ns) DropOutgoingHandshake:456(302ns) SwitchToState:178(114ns) ProcessPendingSessionEvents:495(5775513ns) {ForwardSessionEventToSession:612(714773ns) ForwardSessionEventToSession:612(10649ns) ForwardSessionEventToSession:612(696941ns) ForwardSessionEventToSession:612(1915590ns) ForwardSessionEventToSession:612(3787ns) ForwardSessionEventToSession:612(2465ns) ForwardSessionEventToSession:612(1750594ns) ForwardSessionEventToSession:612(216749ns) ForwardSessionEventToSession:612(3620ns) ForwardSessionEventToSession:612(448169ns)}}}
2025-11-11T17:21:21.299447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519165733636816:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.299496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:21.299562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519165733636816:2108] [node 5] ICP22 created new session: [2:7571519170028604144:2108]
2025-11-11T17:21:21.299612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519170028604144:2108] [node 5] ICS09 handshake done sender: [2:7571519170028604118:2111] self: [2:7571519170028604117:0] peer: [5:7571519168356070875:0] socket: 633 qp: -1
2025-11-11T17:21:21.299643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519170028604144:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:21.299672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:21.300738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519170028604130:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:21.301761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519165733636752:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.301788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:21.301815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519165733636752:2048] [node 1] ICP22 created new session: [2:7571519170028604146:2048]
2025-11-11T17:21:21.301850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519170028604146:2048] [node 1] ICS09 handshake done sender: [2:7571519170028604130:2117] self: [2:7571519170028604131:0] peer: [1:7571519168050839631:0] socket: 649 qp: -1
2025-11-11T17:21:21.301877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519170028604146:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:21.301897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:21.208011Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028913:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.211691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519169829028891:2107] [node 4] ICP18 handshake (actor [3:7571519169829028913:2118]) is too old
2025-11-11T17:21:21.212351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519169829028892:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.214159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.297972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519164061103493:2048] [node 1] ICP22 created new session: [5:7571519168356070884:2048]
2025-11-11T17:21:21.293909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:21.310836Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519166943183120:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.218916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519169829028892:2108] [node 5] ICP22 created new session: [3:7571519169829028919:2108]
2025-11-11T17:21:21.307282Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519169735883203:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.293951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519168050839630:2109] [node 6] ICP22 created new session: [1:7571519168050839665:2109]
2025-11-11T17:21:21.298071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519168356070883:2050] [node 3] ICS09 handshake done sender: [5:7571519168356070860:2113] self: [5:7571519168356070869:0] peer: [3:7571519169829028896:0] socket: 626 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.313820Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.318449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519162648215729:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519166943183114:2121] held: no
2025-11-11T17:21:21.318831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519166943183091:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.319205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519166943183119:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.321182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519162648215792:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.218978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519169829028919:2108] [node 5] ICS09 handshake done sender: [3:7571519169829028897:2112] self: [3:7571519169829028896:0] peer: [5:7571519168356070869:0] socket: 619 qp: -1
2025-11-11T17:21:21.298985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519168356070851:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.309630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883209:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
2025-11-11T17:21:21.321360Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604127:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.321399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519170028604127:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.293992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519168050839665:2109] [node 6] ICS09 handshake done sender: [1:7571519168050839636:2112] self: [1:7571519168050839635:0] peer: [6:7571519166943183117:0] socket: 642 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.219021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519169829028919:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:21.314133Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519169735883209:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.329561Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519170028604127:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.328129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183121:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.299013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:21.298175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519168050839665:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:21.219070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:21.338396Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519166943183114:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:21.319178Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519169735883189:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.300292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519168356070884:2048] [node 1] ICS09 handshake done sender: [5:7571519168356070868:2117] self: [5:7571519168356070876:0] peer: [1:7571519168050839637:0] socket: 650 qp: -1
2025-11-11T17:21:21.298917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:21.342597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519170028604113:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.230364Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028911:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.246093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519169829028892:2108] [node 5] ICP18 handshake (actor [3:7571519169829028911:2117]) is too old
2025-11-11T17:21:21.269525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519169829028921:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14127
2025-11-11T17:21:21.281919Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028921:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.283529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519165534061531:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.321708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519169829028924:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14127
2025-11-11T17:21:21.339876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519169829028923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14127
2025-11-11T17:21:21.342860Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519169829028921:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:21.349347Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.352197Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519169829028923:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.352374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519165534061532:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.319195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519169735883189:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.300771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519168356070883:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:21.300811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:21.301266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519168356070884:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:21.301464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:21.301717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519168356070851:2107] [node 4] ICP22 created new session: [5:7571519168356070885:2107]
2025-11-11T17:21:21.303442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519168356070885:2107] [node 4] ICS09 handshake done sender: [5:7571519168356070873:2120] self: [5:7571519168356070877:0] peer: [4:7571519169735883186:0] socket: 668 qp: -1
2025-11-11T17:21:21.303504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519168356070885:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:21.303578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:21.305733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519164061103494:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.305791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:21.305834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519164061103494:2049] [node 2] ICP22 created new session: [5:7571519168356070889:2049]
2025-11-11T17:21:21.305890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519168356070889:2049] [node 2] ICS09 handshake done sender: [5:7571519168356070864:2116] self: [5:7571519168356070875:0] peer: [2:7571519170028604117:0] socket: 637 qp: -1
2025-11-11T17:21:21.305939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519168356070889:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:21.306003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:21.309139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519164061103495:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(17997740ns) {HandleHandshakeStatus:329(17996293ns) {DropIncomingHandshake:441(1331ns) DropOutgoingHandshake:456(134ns) SwitchToState:178(461ns) ProcessPendingSessionEvents:495(1358ns)}}
2025-11-11T17:21:21.311224Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.312276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519164061103494:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:21.321365Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519168356070871:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.321380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519168356070871:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.337836Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519168356070871:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:21.342858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519168356070852:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.349722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:21.349772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519168356070852:2108] [node 6] ICP22 created new session: [5:7571519168356070892:2108]
2025-11-11T17:21:21.349818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519168356070892:2108] [node 6] ICS09 handshake done sender: [5:7571519168356070871:2118] self: [5:7571519168356070870:0] peer: [6:7571519166943183123:0] socket: 673 qp: -1
2025-11-11T17:21:21.349862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519168356070892:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:21.349897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:21.339380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183124:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.348768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:21.321000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519165440915819:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:21.339474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519166943183125:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40442
2025-11-11T17:21:21.368001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519165534061531:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519169829028921:2119] held: no
2025-11-11T17:21:21.310892Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519168050839640:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:21.351852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519162648215729:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.351785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519170028604113:2107] [node 4] ICP22 created new session: [2:7571519170028604149:2107]
2025-11-11T17:21:21.326838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519169735883212:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23621
2025-11-11T17:21:21.308937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519163755872333:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(19549290ns) {HandleHandshakeStatus:329(19547036ns) {DropIncomingHandshake:441(13088ns) DropOutgoingHandshake:456(353ns) SwitchToState:178(332ns) ProcessPendingSessionEvents:495(8290492ns) {ForwardSessionEventToSession:612(1691399ns) ForwardSessionEventToSession:612(474759ns) ForwardSessionEventToSession:612(3597ns) ForwardSessionEventToSession:612(3769ns) ForwardSessionEventToSession:612(1801214ns) ForwardSessionEventToSession:612(3679ns) ForwardSessionEventToSession:612(2890186ns) ForwardSessionEventToSession:612(7481ns) ForwardSessionEventToSession:612(3065ns) ForwardSessionEventToSession:612(1393645ns)}}}
2025-11-11T17:21:21.358009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519170028604149:2107] [node 4] ICS09 handshake done sender: [2:7571519170028604127:2115] self: [2:7571519170028604126:0] peer: [4:7571519169735883211:0] socket: 680 qp: -1
2025-11-11T17:21:21.351896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:21.339787Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519169735883189:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:21.328340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519163755872270:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.338433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:21.338489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519163755872270:2049] [node 3] ICP22 created new session: [1:7571519168050839668:2049]
2025-11-11T17:21:21.349282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519168050839668:2049] [node 3] ICS09 handshake done sender: [1:7571519168050839640:2114] self: [1:7571519168050839639:0] peer: [3:7571519169829028922:0] socket: 670 qp: -1
2025-11-11T17:21:21.352997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519168050839668:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:21.353040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:21.410918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519163755872270:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(76362823ns) {HandleHandshakeStatus:329(76360835ns) {DropIncomingHandshake:441(2677892ns) DropOutgoingHandshake:456(3545ns) SwitchToState:178(831ns) ProcessPendingSessionEvents:495(16468601ns) {ForwardSessionEventToSession:612(8342069ns) ForwardSessionEventToSession:612(8185ns) ForwardSessionEventToSession:612(5377768ns) ForwardSessionEventToSession:612(4945ns) ForwardSessionEventToSession:612(2695973ns) ForwardSessionEventToSession:612(6868ns) ForwardSessionEventToSession:612(4243ns) ForwardSessionEventToSession:612(5197ns) ForwardSessionEventToSession:612(3409ns) ForwardSessionEventToSession:612(3067ns)}}}
2025-11-11T17:21:21.400606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519165534061531:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(25755446ns) {HandleHandshakeStatus:395(25198568ns) {LogHandshakeFail:472(5088881ns) {UpdateErrorStateLog:544(12024ns)} DropOutgoingHandshake:456(2939ns) ProcessPendingSessionEvents:495(9944052ns) {DropSessionEvent:512(23150ns) DropSessionEvent:512(6625ns) DropSessionEvent:512(4915ns) DropSessionEvent:512(5922ns) DropSessionEvent:512(5051374ns) DropSessionEvent:512(8718ns) DropSessionEvent:512(6531ns) DropSessionEvent:512(19747ns) DropSessionEvent:512(4707ns) DropSessionEvent:512(2933050ns)}}}
2025-11-11T17:21:21.408554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519165534061531:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.414082Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.415565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519165534061532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519169829028923:2120] held: no
2025-11-11T17:21:21.348677Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:21.350733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519169735883177:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:21.350801Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519169735883209:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:21.366348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519165440915819:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.366408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:21.366462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519165440915819:2049] [node 2] ICP22 created new session: [4:7571519169735883214:2049]
2025-11-11T17:21:21.366517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519169735883214:2049] [node 2] ICS09 handshake done sender: [4:7571519169735883209:2121] self: [4:7571519169735883211:0] peer: [2:7571519170028604126:0] socket: 681 qp: -1
2025-11-11T17:21:21.366565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519169735883214:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:21.366629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:21.367038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519169735883178:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.367090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:21.367795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519169735883178:2109] [node 6] ICP22 created new session: [4:7571519169735883216:2109]
2025-11-11T17:21:21.371366Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.381722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519169735883216:2109] [node 6] ICS09 handshake done sender: [4:7571519169735883189:2115] self: [4:7571519169735883188:0] peer: [6:7571519166943183122:0] socket: 674 qp: -1
2025-11-11T17:21:21.381763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519169735883216:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:21.386375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519165440915819:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:21.390079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:21.419541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519169735883178:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(52204657ns) {HandleHandshakeStatus:329(52202697ns) {DropIncomingHandshake:441(15662ns) DropOutgoingHandshake:456(153ns) SwitchToState:178(559ns) ProcessPendingSessionEvents:495(13211976ns) {ForwardSessionEventToSession:612(5730731ns) ForwardSessionEventToSession:612(2225927ns) ForwardSessionEventToSession:612(4328ns) ForwardSessionEventToSession:612(5202942ns) ForwardSessionEventToSession:612(6056ns) ForwardSessionEventToSession:612(3200ns) ForwardSessionEventToSession:612(7003ns) ForwardSessionEventToSession:612(2391ns) ForwardSessionEventToSession:612(5693ns) ForwardSessionEventToSession:612(5570ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:21.353439Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18620) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.368007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519162648215729:2048] [node 1] ICP22 created new session: [6:7571519166943183126:2048]
2025-11-11T17:21:21.371086Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519166943183119:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:21.373308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519166943183126:2048] [node 1] ICS09 handshake done sender: [6:7571519166943183114:2121] self: [6:7571519166943183117:0] peer: [1:7571519168050839635:0] socket: 654 qp: -1
2025-11-11T17:21:21.374140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519162648215792:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18620) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519166943183119:2122] held: no
2025-11-11T17:21:21.374213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519162648215792:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.374242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:21.374273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519162648215792:2106] [node 5] ICP22 created new session: [6:7571519166943183127:2106]
2025-11-11T17:21:21.374306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519166943183127:2106] [node 5] ICS09 handshake done sender: [6:7571519166943183119:2122] self: [6:7571519166943183123:0] peer: [5:7571519168356070870:0] socket: 676 qp: -1
2025-11-11T17:21:21.374335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519166943183127:2106] [node 5] ICS10 traffic start
2025-11-11T17:21:21.374357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:21.378789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519166943183126:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:21.382042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519166943183120:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:21.385408Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17756) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:21.386890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:21.398614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519166943183091:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17756) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519166943183120:2123] held: no
2025-11-11T17:21:21.398737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519166943183091:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.398779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:21.398829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519166943183091:2110] [node 4] ICP22 created new session: [6:7571519166943183130:2110]
2025-11-11T17:21:21.398889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519166943183130:2110] [node 4] ICS09 handshake done sender: [6:7571519166943183120:2123] self: [6:7571519166943183122:0] peer: [4:7571519169735883188:0] socket: 677 qp: -1
2025-11-11T17:21:21.398923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519166943183130:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:21.398968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:21.411648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519162648215729:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(56430658ns) {HandleHandshakeStatus:329(56428098ns) {DropIncomingHandshake:441(602ns) DropOutgoingHandshake:456(439ns) SwitchToState:178(908ns) ProcessPendingSessionEvents:495(1033ns)}}
2025-11-11T17:21:21.367648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519170028604149:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:21.367694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:21.386573Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519170028604120:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:21.386616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519170028604120:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:21.398064Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519170028604120:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:21.421937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519169829028926:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14127
2025-11-11T17:21:21.422434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:21.422492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519165534061531:2048] [node 1] ICP22 created new session: [3:7571519169829028927:2048]
2025-11-11T17:21:21.422543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519169829028927:2048] [node 1] ICS09 handshake done sender: [3:7571519169829028921:2119] self: [3:7571519169829028922:0] peer: [1:7571519168050839639:0] socket: 671 qp: -1
2025-11-11T17:21:21.422584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519169829028927:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:21.422617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:21.422820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519165534061531:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16455197ns) {HandleHandshakeStatus:329(16452889ns) {DropIncomingHandshake:441(1189ns) DropOutgoingHandshake:456(496ns) SwitchToState:178(540ns) ProcessPendingSessionEvents:495(535ns)}}
2025-11-11T17:21:21.398555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519165733636753:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:21.398636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:21.398705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519165733636753:2049] [node 3] ICP22 created new session: [2:7571519170028604152:2049]
2025-11-11T17:21:21.398771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519170028604152:2049] [node 3] ICS09 handshake done sender: [2:7571519170028604120:2112] self: [2:7571519170028604119:0] peer: [3:7571519169829028925:0] socket: 682 qp: -1
2025-11-11T17:21:21.399812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519170028604113:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(58840285ns) {HandleHandshakeStatus:329(57031923ns) {DropIncomingHandshake:441(16750ns) DropOutgoingHandshake:456(399ns) SwitchToState:178(501ns) ProcessPendingSessionEvents:495(12925053ns) {ForwardSessionEventToSession:612(12868511ns) ForwardSessionEventToSession:612(4615ns) ForwardSessionEventToSession:612(4877ns) ForwardSessionEventToSession:612(7184ns) ForwardSessionEventToSession:612(3764ns) ForwardSessionEventToSession:612(8046ns) ForwardSessionEventToSession:612(3434ns) ForwardSessionEventToSession:612(5545ns) ForwardSessionEventToSession:612(6298ns) ForwardSessionEventToSession:612(3247ns)}}}
2025-11-11T17:21:21.405866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519170028604152:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:21.405922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:21.418161Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519165733636753:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(19287361ns) {HandleHandshakeStatus:329(19284212ns) {DropIncomingHandshake:441(17959ns) DropOutgoingHandshake:456(597ns) SwitchToState:178(477ns) ProcessPendingSessionEvents:495(7531077ns) {ForwardSessionEventToSession:612(7474529ns) ForwardSessionEventToSession:612(10570ns) ForwardSessionEventToSession:612(6641ns) ForwardSessionEventToSession:612(2725ns) ForwardSessionEventToSession:612(2564ns) ForwardSessionEventToSession:612(5869ns) ForwardSessionEventToSession:612(3385ns) ForwardSessionEventToSession:612(2551ns) ForwardSessionEventToSession:612(5033ns) ForwardSessionEventToSession:612(4912ns)}}}
2025-11-11T17:21:21.425725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519169829028923:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:21.426125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519165534061532:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:21.426180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:21.426233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519165534061532:2049] [node 2] ICP22 created new session: [3:7571519169829028929:2049]
2025-11-11T17:21:21.426284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519169829028929:2049] [node 2] ICS09 handshake done sender: [3:7571519169829028923:2120] self: [3:7571519169829028925:0] peer: [2:7571519170028604119:0] socket: 683 qp: -1
2025-11-11T17:21:21.426322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519169829028929:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:21.426363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:21.460493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:21.460825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519170028604146:2048] [node 1] ICS01 socket: 649 reason# EndOfStream
2025-11-11T17:21:21.460875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519165733636752:2048] [node 1] ICP30 unregister session Session# [2:7571519170028604146:2048] VirtualId# [2:7571519170028604131:0]
2025-11-11T17:21:21.460913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519170028604146:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.460995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:21.461038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:21.461771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:21.461947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519169829028927:2048] [node 1] ICS01 socket: 671 reason# EndOfStream
2025-11-11T17:21:21.461988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519165534061531:2048] [node 1] ICP30 unregister session Session# [3:7571519169829028927:2048] VirtualId# [3:7571519169829028922:0]
2025-11-11T17:21:21.462022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519169829028927:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.462074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:21.462103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:21.464995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:21.465234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519169735883201:2048] [node 1] ICS01 socket: 627 reason# EndOfStream
2025-11-11T17:21:21.465281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519165440915818:2048] [node 1] ICP30 unregister session Session# [4:7571519169735883201:2048] VirtualId# [4:7571519169735883195:0]
2025-11-11T17:21:21.465311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519169735883201:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.465388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:21.465424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:21.466115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:21.466299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519168356070884:2048] [node 1] ICS01 socket: 650 reason# EndOfStream
2025-11-11T17:21:21.466341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519164061103493:2048] [node 1] ICP30 unregister session Session# [5:7571519168356070884:2048] VirtualId# [5:7571519168356070876:0]
2025-11-11T17:21:21.466366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519168356070884:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.466422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:21.466466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:21.467134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:21.467302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519166943183126:2048] [node 1] ICS01 socket: 654 reason# EndOfStream
2025-11-11T17:21:21.467338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519162648215729:2048] [node 1] ICP30 unregister session Session# [6:7571519166943183126:2048] VirtualId# [6:7571519166943183117:0]
2025-11-11T17:21:21.467361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519166943183126:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.467434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:21.467476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:21.517642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:21.517866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519169829028929:2049] [node 2] ICS01 socket: 683 reason# EndOfStream
2025-11-11T17:21:21.517905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519165534061532:2049] [node 2] ICP30 unregister session Session# [3:7571519169829028929:2049] VirtualId# [3:7571519169829028925:0]
2025-11-11T17:21:21.517936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519169829028929:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.518008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:21.518045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:21.525772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:21.525967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519168356070889:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-11-11T17:21:21.526010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519164061103494:2049] [node 2] ICP30 unregister session Session# [5:7571519168356070889:2049] VirtualId# [5:7571519168356070875:0]
2025-11-11T17:21:21.526043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519168356070889:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.526115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:21.526162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:21.525778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:21.525986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519169735883214:2049] [node 2] ICS01 socket: 681 reason# EndOfStream
2025-11-11T17:21:21.526043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519165440915819:2049] [node 2] ICP30 unregister session Session# [4:7571519169735883214:2049] VirtualId# [4:7571519169735883211:0]
2025-11-11T17:21:21.529800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:21.526075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519169735883214:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.526168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:21.526210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:21.530093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519166943183115:2049] [node 2] ICS01 socket: 635 reason# EndOfStream
2025-11-11T17:21:21.530171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519162648215730:2049] [node 2] ICP30 unregister session Session# [6:7571519166943183115:2049] VirtualId# [6:7571519166943183109:0]
2025-11-11T17:21:21.530230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519166943183115:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.530390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:21.531372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:21.601363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:21.601650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519169735883197:2050] [node 3] ICS01 socket: 622 reason# EndOfStream
2025-11-11T17:21:21.601699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519165440915820:2050] [node 3] ICP30 unregister session Session# [4:7571519169735883197:2050] VirtualId# [4:7571519169735883194:0]
2025-11-11T17:21:21.601737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519169735883197:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.601820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:21.601882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:21.602598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:21.602804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519168356070883:2050] [node 3] ICS01 socket: 626 reason# EndOfStream
2025-11-11T17:21:21.602870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519164061103495:2050] [node 3] ICP30 unregister session Session# [5:7571519168356070883:2050] VirtualId# [5:7571519168356070869:0]
2025-11-11T17:21:21.602905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519168356070883:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.602966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:21.603005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:21.603562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:21.603748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519166943183110:2050] [node 3] ICS01 socket: 621 reason# EndOfStream
2025-11-11T17:21:21.603779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519162648215731:2050] [node 3] ICP30 unregister session Session# [6:7571519166943183110:2050] VirtualId# [6:7571519166943183106:0]
2025-11-11T17:21:21.603804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519166943183110:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.603865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:21.603895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:21.652267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:21.652537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519166943183130:2110] [node 4] ICS01 socket: 677 reason# EndOfStream
2025-11-11T17:21:21.652634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519166943183091:2110] [node 4] ICP30 unregister session Session# [6:7571519166943183130:2110] VirtualId# [6:7571519166943183122:0]
2025-11-11T17:21:21.652697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519166943183130:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.652815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:21.652874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:21.653841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:21.654155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519168356070885:2107] [node 4] ICS07 socket disconnect 668 reason# EndOfStream
2025-11-11T17:21:21.654222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519168356070885:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.654335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:21.654422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519168356070885:2107] [node 4] ICS15 start handshake
2025-11-11T17:21:21.655253Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17756) Couldn't connect to any resolved address
2025-11-11T17:21:21.656251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519168356070851:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17756) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:21.656319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519168356070885:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:21.656356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519168356070851:2107] [node 4] ICP30 unregister session Session# [5:7571519168356070885:2107] VirtualId# [5:7571519168356070877:0]
2025-11-11T17:21:21.656402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:21.656443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519168356070851:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:21.656462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:21.667177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519168356070851:2107] [node 4] ICP33 wake up from error state
2025-11-11T17:21:21.713224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:21.713470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519166943183127:2106] [node 5] ICS01 socket: 676 reason# EndOfStream
2025-11-11T17:21:21.713559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519162648215792:2106] [node 5] ICP30 unregister session Session# [6:7571519166943183127:2106] VirtualId# [6:7571519166943183123:0]
2025-11-11T17:21:21.713632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519166943183127:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:21.713785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:21.713870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:21.827444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519167305441174:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:21.827491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519167305441175:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:21.856238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519169486761374:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:21.856266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519169486761375:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:21.876520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519170351115373:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:21.876671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519170351115374:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:21.895848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519167850850769:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:21.895888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519167850850770:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:21.895907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519167850850771:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:21.918508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519168354482495:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:21.918536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519168354482496:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:21.918553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519168354482497:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:21.938336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519167813293851:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:21.938376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519167813293852:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:21.938535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519167813293853:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:22.050692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519171600408533:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:22.079686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519172649449854:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:22.161147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519172145818128:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:22.127028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519172108261208:2105] [node 5] ICP01 ready to work
2025-11-11T17:21:22.145633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519174646082732:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:22.129721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519171600408535:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:22.160776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519173781728743:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:22.151298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519174646082733:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:22.167082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519173781728744:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:22.167176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519172108261211:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:22.168664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519172649449858:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:22.162358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519171600408536:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:22.177290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519172145818135:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:22.167063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519174646082734:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:22.171916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519170351115373:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(126634855ns) {Configure:104(126629218ns) {ProcessConfigured:135(60078ns) {StartInitialHandshake:157(57095ns) {DropHandshakes:469(2297ns) {DropIncomingHandshake:441(391ns) DropOutgoingHandshake:456(426ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:476(1763ns) {SwitchToState:178(594ns)}}}}}
2025-11-11T17:21:22.177160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519173781728745:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:22.185609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519172649449854:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(36967275ns) {Configure:104(36957988ns) {ProcessConfigured:135(54935ns) {StartInitialHandshake:157(52114ns) {DropHandshakes:469(4877ns) {DropIncomingHandshake:441(424ns) DropOutgoingHandshake:456(361ns)} GenerateSessionVirtualId:988(627ns) PrepareNewSessionHandshake:476(1505ns) {SwitchToState:178(453ns)}}}}}
2025-11-11T17:21:22.201125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519172108261208:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32040979ns) {Configure:104(32035553ns) {ProcessConfigured:135(47171ns) {StartInitialHandshake:157(44839ns) {DropHandshakes:469(1745ns) {DropIncomingHandshake:441(289ns) DropOutgoingHandshake:456(376ns)} GenerateSessionVirtualId:988(987ns) PrepareNewSessionHandshake:476(1070ns) {SwitchToState:178(401ns)}}}}}
2025-11-11T17:21:22.197714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519167305441175:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(141840864ns) {Configure:104(141835052ns) {ProcessConfigured:135(47046ns) {StartInitialHandshake:157(45046ns) {DropHandshakes:469(2269ns) {DropIncomingHandshake:441(624ns) DropOutgoingHandshake:456(520ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:476(1645ns) {SwitchToState:178(586ns)}}}}}
2025-11-11T17:21:22.195354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519169486761374:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25382891ns) {Configure:104(25376529ns) {ProcessConfigured:135(75268ns) {StartInitialHandshake:157(72485ns) {DropHandshakes:469(2453ns) {DropIncomingHandshake:441(780ns) DropOutgoingHandshake:456(353ns)} GenerateSessionVirtualId:988(1381ns) PrepareNewSessionHandshake:476(1820ns) {SwitchToState:178(484ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.222919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519167850850771:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(54711069ns) {Configure:104(54702876ns) {ProcessConfigured:135(60022ns) {StartInitialHandshake:157(57316ns) {DropHandshakes:469(2183ns) {DropIncomingHandshake:441(446ns) DropOutgoingHandshake:456(446ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:476(1337ns) {SwitchToState:178(546ns)}}}}}
2025-11-11T17:21:22.205914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519174646082734:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29567826ns) {Configure:104(29564168ns) {ProcessConfigured:135(52168ns) {StartInitialHandshake:157(50073ns) {DropHandshakes:469(2610ns) {DropIncomingHandshake:441(747ns) DropOutgoingHandshake:456(406ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:476(1123ns) {SwitchToState:178(310ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.269964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519167305441174:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(195376314ns) {Configure:104(195367888ns) {ProcessConfigured:135(11273823ns) {StartInitialHandshake:157(11270563ns) {DropHandshakes:469(2608ns) {DropIncomingHandshake:441(487ns) DropOutgoingHandshake:456(428ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:476(6786ns) {SwitchToState:178(2202ns)}}}}}
2025-11-11T17:21:22.272158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519169486761375:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(103509250ns) {Configure:104(103504043ns) {ProcessConfigured:135(17546567ns) {StartInitialHandshake:157(13431266ns) {DropHandshakes:469(3732ns) {DropIncomingHandshake:441(625ns) DropOutgoingHandshake:456(556ns)} GenerateSessionVirtualId:988(1830ns) PrepareNewSessionHandshake:476(3770ns) {SwitchToState:178(1734ns)}}}}}
2025-11-11T17:21:22.217854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519167813293852:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(47871573ns) {Configure:104(42121703ns) {ProcessConfigured:135(79560ns) {StartInitialHandshake:157(76780ns) {DropHandshakes:469(2497ns) {DropIncomingHandshake:441(477ns) DropOutgoingHandshake:456(623ns)} GenerateSessionVirtualId:988(877ns) PrepareNewSessionHandshake:476(1815ns) {SwitchToState:178(827ns)}}}}}
2025-11-11T17:21:22.209756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519174646082733:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(50972985ns) {Configure:104(50968530ns) {ProcessConfigured:135(49118ns) {StartInitialHandshake:157(46942ns) {DropHandshakes:469(2836ns) {DropIncomingHandshake:441(449ns) DropOutgoingHandshake:456(861ns)} GenerateSessionVirtualId:988(319ns) PrepareNewSessionHandshake:476(957ns) {SwitchToState:178(327ns)}}}}}
2025-11-11T17:21:22.230396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449862:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.218676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261218:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.242141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519172145818136:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7185
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.302539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519171600408535:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(134669923ns) {Configure:104(134665702ns) {ProcessConfigured:135(67058ns) {StartInitialHandshake:157(64009ns) {DropHandshakes:469(4919ns) {DropIncomingHandshake:441(489ns) DropOutgoingHandshake:456(503ns)} GenerateSessionVirtualId:988(1125ns) PrepareNewSessionHandshake:476(1703ns) {SwitchToState:178(633ns)}}}}}
2025-11-11T17:21:22.302691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519171600408542:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58492
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.230461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449863:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.279903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519174646082732:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(128260283ns) {Configure:104(128254542ns) {ProcessConfigured:135(17833546ns) {StartInitialHandshake:157(17830807ns) {DropHandshakes:469(2645ns) {DropIncomingHandshake:441(782ns) DropOutgoingHandshake:456(443ns)} GenerateSessionVirtualId:988(743ns) PrepareNewSessionHandshake:476(4751ns) {SwitchToState:178(2271ns)}}}}}
2025-11-11T17:21:22.271599Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519172145818136:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.218734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261219:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.282368Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519172649449862:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.306341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519173781728752:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18532
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.282549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519174646082740:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60452
2025-11-11T17:21:22.306269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519171600408541:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58492
2025-11-11T17:21:22.294655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519172145818135:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(72231439ns) {Configure:104(72223258ns) {ProcessConfigured:135(30592413ns) {StartInitialHandshake:157(30589494ns) {DropHandshakes:469(1901ns) {DropIncomingHandshake:441(309ns) DropOutgoingHandshake:456(347ns)} GenerateSessionVirtualId:988(478ns) PrepareNewSessionHandshake:476(5186ns) {SwitchToState:178(2349ns)}}}}}
2025-11-11T17:21:22.259121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519167813293851:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(40724899ns) {Configure:104(40718798ns) {ProcessConfigured:135(12737100ns) {StartInitialHandshake:157(12733725ns) {DropHandshakes:469(2682ns) {DropIncomingHandshake:441(743ns) DropOutgoingHandshake:456(626ns)} GenerateSessionVirtualId:988(1731ns) PrepareNewSessionHandshake:476(2999ns) {SwitchToState:178(1316ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.314483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519173781728753:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18532
2025-11-11T17:21:22.322067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519174646082750:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60452
2025-11-11T17:21:22.338836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519168354482496:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(136049323ns) {Configure:104(136042488ns) {ProcessConfigured:135(20236345ns) {StartInitialHandshake:157(20233242ns) {DropHandshakes:469(2576ns) {DropIncomingHandshake:441(599ns) DropOutgoingHandshake:456(426ns)} GenerateSessionVirtualId:988(988ns) PrepareNewSessionHandshake:476(4769ns) {SwitchToState:178(2522ns)}}}}}
2025-11-11T17:21:22.339058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519168354482495:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(186721072ns) {Configure:104(186713887ns) {ProcessConfigured:135(20848721ns) {StartInitialHandshake:157(20845707ns) {DropHandshakes:469(1762ns) {DropIncomingHandshake:441(311ns) DropOutgoingHandshake:456(299ns)} GenerateSessionVirtualId:988(978ns) PrepareNewSessionHandshake:476(3996ns) {SwitchToState:178(1278ns)}}}}}
2025-11-11T17:21:22.314039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519171600408543:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58492
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.273315Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519172108261218:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.323635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408542:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.337848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519172108261211:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(121402048ns) {Configure:104(121398007ns) {ProcessConfigured:135(11752622ns) {StartInitialHandshake:157(11748795ns) {DropHandshakes:469(2501ns) {DropIncomingHandshake:441(565ns) DropOutgoingHandshake:456(290ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:476(3625ns) {SwitchToState:178(1409ns)}}}}}
2025-11-11T17:21:22.349035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519172108261211:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.358225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519172108261219:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.322335Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082740:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.337235Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728752:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.349530Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519172145818132:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.344595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519168354482497:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.349564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519172145818132:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.343481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519173781728743:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(167175944ns) {Configure:104(167169574ns) {ProcessConfigured:135(14485044ns) {StartInitialHandshake:157(14481450ns) {DropHandshakes:469(2820ns) {DropIncomingHandshake:441(444ns) DropOutgoingHandshake:456(705ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(3795ns) {SwitchToState:178(1036ns)}}}}}
2025-11-11T17:21:22.365116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519174646082747:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60452
2025-11-11T17:21:22.323725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408541:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.365182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519167813293853:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(160598174ns) {Configure:104(160593056ns) {ProcessConfigured:135(18974908ns) {StartInitialHandshake:157(18973300ns) {DropHandshakes:469(2099ns) {DropIncomingHandshake:441(461ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(657ns) PrepareNewSessionHandshake:476(1692ns) {SwitchToState:178(514ns)}}}}}
2025-11-11T17:21:22.357136Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519172649449863:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.343761Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728753:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.365991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082743:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.328725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519171600408544:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58492
2025-11-11T17:21:22.357706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449870:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.354709Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519172145818132:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:22.365542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519167813293852:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.344540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519173781728757:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18532
2025-11-11T17:21:22.366023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519174646082743:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.358081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519167850850770:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(152745435ns) {Configure:104(152739468ns) {ProcessConfigured:135(26969719ns) {StartInitialHandshake:157(26967331ns) {DropHandshakes:469(2275ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(377ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:476(1352ns) {SwitchToState:178(269ns)}}}}}
2025-11-11T17:21:22.369649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261228:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.337579Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408543:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.357830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449871:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.377011Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519172108261218:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:22.365890Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728749:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.386242Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519174646082743:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:22.353999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519171600408533:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(162172554ns) {Configure:104(162165012ns) {ProcessConfigured:135(15818221ns) {StartInitialHandshake:157(15815695ns) {DropHandshakes:469(3036ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(473ns)} GenerateSessionVirtualId:988(705ns) PrepareNewSessionHandshake:476(3153ns) {SwitchToState:178(1115ns)}}}}}
2025-11-11T17:21:22.365159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519172145818128:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.357880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449872:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.377309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519172108261211:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.358435Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408544:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.365914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519173781728749:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.394977Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082747:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.362214Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519172649449858:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(181667786ns) {Configure:104(179415947ns) {ProcessConfigured:135(11205370ns) {StartInitialHandshake:157(11202215ns) {DropHandshakes:469(3545234ns) {DropIncomingHandshake:441(2300ns) DropOutgoingHandshake:456(1344ns)} GenerateSessionVirtualId:988(2457ns) PrepareNewSessionHandshake:476(3998ns) {SwitchToState:178(1394ns)}}}}}
2025-11-11T17:21:22.366091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519172649449872:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.379129Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519172649449870:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.386381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519172649449854:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.402496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519168354482495:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.419647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449879:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.420637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449878:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.422000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449880:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.422484Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519172649449872:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:22.422676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519172649449854:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.422724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:22.424500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519172649449854:2108] [node 4] ICP22 created new session: [5:7571519172649449881:2108]
2025-11-11T17:21:22.449843Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519172649449871:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.449919Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519172649449862:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:22.454405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519172649449881:2108] [node 4] ICS09 handshake done sender: [5:7571519172649449872:2118] self: [5:7571519172649449876:0] peer: [4:7571519172145818139:0] socket: 661 qp: -1
2025-11-11T17:21:22.455452Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.456935Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519172649449870:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:22.462124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519172649449881:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:22.464153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519168354482497:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.464223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:22.464283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519168354482497:2050] [node 3] ICP22 created new session: [5:7571519172649449883:2050]
2025-11-11T17:21:22.464332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519172649449883:2050] [node 3] ICS09 handshake done sender: [5:7571519172649449862:2112] self: [5:7571519172649449868:0] peer: [3:7571519174646082742:0] socket: 640 qp: -1
2025-11-11T17:21:22.464372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519172649449883:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:22.464440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:22.464767Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519168354482497:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:22.465125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:22.377358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:22.366015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.476711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519168354482495:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.476776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:22.476816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519168354482495:2048] [node 1] ICP22 created new session: [5:7571519172649449886:2048]
2025-11-11T17:21:22.476854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519172649449886:2048] [node 1] ICS09 handshake done sender: [5:7571519172649449870:2116] self: [5:7571519172649449877:0] peer: [1:7571519171600408547:0] socket: 658 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.379205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728757:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.417751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082750:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.365803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519171600408536:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(157688597ns) {Configure:104(157682072ns) {ProcessConfigured:135(65415ns) {StartInitialHandshake:157(63396ns) {DropHandshakes:469(1106ns) {DropIncomingHandshake:441(307ns) DropOutgoingHandshake:456(203ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:476(628ns) {SwitchToState:178(256ns)}}}}}
2025-11-11T17:21:22.376156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519172145818128:2109] [node 6] ICP22 created new session: [4:7571519172145818147:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.377406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519172108261211:2109] [node 4] ICP22 created new session: [6:7571519172108261230:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.417926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519170351115373:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.476885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519172649449886:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:22.385802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519173781728744:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(191512864ns) {Configure:104(191502604ns) {ProcessConfigured:135(10494120ns) {StartInitialHandshake:157(10490765ns) {DropHandshakes:469(753689ns) {DropIncomingHandshake:441(531ns) DropOutgoingHandshake:456(383ns)} GenerateSessionVirtualId:988(899ns) PrepareNewSessionHandshake:476(3592ns) {SwitchToState:178(1463ns)}}}}}
2025-11-11T17:21:22.379655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519172145818147:2109] [node 6] ICS09 handshake done sender: [4:7571519172145818132:2111] self: [4:7571519172145818131:0] peer: [6:7571519172108261227:0] socket: 625 qp: -1
2025-11-11T17:21:22.402144Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408540:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.377448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519172108261230:2109] [node 4] ICS09 handshake done sender: [6:7571519172108261218:2113] self: [6:7571519172108261227:0] peer: [4:7571519172145818131:0] socket: 635 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.435052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519174646082734:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.476918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.402190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519171600408540:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.398204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519169486761374:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.398776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519173781728749:2121] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:22.476286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519173781728766:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18532
2025-11-11T17:21:22.481291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519173781728745:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.484446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:22.484498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519173781728745:2109] [node 6] ICP22 created new session: [2:7571519173781728769:2109]
2025-11-11T17:21:22.497761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519173781728769:2109] [node 6] ICS09 handshake done sender: [2:7571519173781728749:2121] self: [2:7571519173781728748:0] peer: [6:7571519172108261229:0] socket: 631 qp: -1
2025-11-11T17:21:22.500234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519173781728768:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18532
2025-11-11T17:21:22.500495Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.502606Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519173781728757:2125] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:22.502868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519169486761374:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519173781728757:2125] held: no
2025-11-11T17:21:22.502970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519169486761374:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.503015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:22.503066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519169486761374:2048] [node 1] ICP22 created new session: [2:7571519173781728771:2048]
2025-11-11T17:21:22.503130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519173781728771:2048] [node 1] ICS09 handshake done sender: [2:7571519173781728757:2125] self: [2:7571519173781728765:0] peer: [1:7571519171600408539:0] socket: 654 qp: -1
2025-11-11T17:21:22.503171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519173781728771:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:22.503210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:22.506031Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728758:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.506076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519173781728758:2126] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.512983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519173781728769:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:22.522089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519173781728767:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18532
2025-11-11T17:21:22.530786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:22.541335Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728768:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.377483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519172108261230:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:22.377520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:22.394327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519172145818147:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-11T17:21:22.477445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519168354482496:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-11T17:21:22.411835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261233:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.441955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:22.549450Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728767:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.402826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408548:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.394416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:22.545748Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519172649449873:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.419715Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519172108261219:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:22.402847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519171600408548:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.555878Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519173781728758:2126] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:22.442017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519174646082734:2108] [node 5] ICP22 created new session: [3:7571519174646082755:2108]
2025-11-11T17:21:22.409420Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519172145818141:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.545792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519172649449873:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.433756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519167813293852:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.557855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519172649449854:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(124312894ns) {HandleHandshakeStatus:329(124310251ns) {DropIncomingHandshake:441(757ns) DropOutgoingHandshake:456(5957ns) SwitchToState:178(455ns) ProcessPendingSessionEvents:495(49898389ns) {ForwardSessionEventToSession:612(14312145ns) ForwardSessionEventToSession:612(1229156ns) ForwardSessionEventToSession:612(4786ns) ForwardSessionEventToSession:612(2774ns) ForwardSessionEventToSession:612(7858170ns) ForwardSessionEventToSession:612(4507ns) ForwardSessionEventToSession:612(10000617ns) ForwardSessionEventToSession:612(2771355ns) ForwardSessionEventToSession:612(15401ns) ForwardSessionEventToSession:612(3445771ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.433819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:22.403752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519171600408548:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:22.442063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519174646082755:2108] [node 5] ICS09 handshake done sender: [3:7571519174646082743:2113] self: [3:7571519174646082742:0] peer: [5:7571519172649449868:0] socket: 634 qp: -1
2025-11-11T17:21:22.409469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519172145818141:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.585854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519173781728743:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.442100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519174646082755:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:22.404122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519171600408535:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.424322Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519172145818141:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:22.558608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519172649449890:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7670
2025-11-11T17:21:22.589707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:22.605670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519173781728743:2108] [node 5] ICP22 created new session: [2:7571519173781728778:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.433892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519167813293852:2049] [node 2] ICP22 created new session: [6:7571519172108261235:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.404168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:22.442140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:22.559511Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519172649449871:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:22.613678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519173781728778:2108] [node 5] ICS09 handshake done sender: [2:7571519173781728758:2126] self: [2:7571519173781728756:0] peer: [5:7571519172649449888:0] socket: 655 qp: -1
2025-11-11T17:21:22.433943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519172108261235:2049] [node 2] ICS09 handshake done sender: [6:7571519172108261219:2114] self: [6:7571519172108261229:0] peer: [2:7571519173781728748:0] socket: 636 qp: -1
2025-11-11T17:21:22.425855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519172145818128:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(58963836ns) {HandleHandshakeStatus:329(58960673ns) {DropIncomingHandshake:441(2300ns) DropOutgoingHandshake:456(616ns) SwitchToState:178(606ns) ProcessPendingSessionEvents:495(133388ns) {ForwardSessionEventToSession:612(74355ns) ForwardSessionEventToSession:612(3843ns) ForwardSessionEventToSession:612(3378ns) ForwardSessionEventToSession:612(7183ns) ForwardSessionEventToSession:612(5711ns) ForwardSessionEventToSession:612(8888ns) ForwardSessionEventToSession:612(2841ns) ForwardSessionEventToSession:612(2880ns) ForwardSessionEventToSession:612(10579ns) ForwardSessionEventToSession:612(2497ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.559725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519168354482496:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.613761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519173781728778:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:22.404212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519171600408535:2108] [node 5] ICP22 created new session: [1:7571519171600408559:2108]
2025-11-11T17:21:22.463802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519172145818135:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.559780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:22.435370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519172108261235:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:22.474419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519174646082758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60452
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.474962Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519174646082750:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:22.613807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.559829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519168354482496:2049] [node 2] ICP22 created new session: [5:7571519172649449891:2049]
2025-11-11T17:21:22.463907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:22.404257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519171600408559:2108] [node 5] ICS09 handshake done sender: [1:7571519171600408548:2117] self: [1:7571519171600408547:0] peer: [5:7571519172649449877:0] socket: 653 qp: -1
2025-11-11T17:21:22.439350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:22.475217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519170351115373:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.404292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519171600408559:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:22.404326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:22.429631Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519171600408540:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:22.434188Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408538:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.434215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519171600408538:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.443499Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519171600408538:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:22.455158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519167305441174:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.455248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:22.455310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519167305441174:2048] [node 2] ICP22 created new session: [1:7571519171600408564:2048]
2025-11-11T17:21:22.455354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519171600408564:2048] [node 2] ICS09 handshake done sender: [1:7571519171600408540:2112] self: [1:7571519171600408539:0] peer: [2:7571519173781728765:0] socket: 652 qp: -1
2025-11-11T17:21:22.455390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519171600408564:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:22.455445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:22.481490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519167305441175:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.484615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:22.484661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519167305441175:2049] [node 3] ICP22 created new session: [1:7571519171600408567:2049]
2025-11-11T17:21:22.506420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519171600408567:2049] [node 3] ICS09 handshake done sender: [1:7571519171600408538:2111] self: [1:7571519171600408537:0] peer: [3:7571519174646082754:0] socket: 650 qp: -1
2025-11-11T17:21:22.513063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519171600408567:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:22.532206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:22.580624Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408552:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.580658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519171600408552:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.590383Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519171600408556:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.590419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519171600408556:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.595335Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519171600408552:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:22.611656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519171600408533:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.619296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519167305441175:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(101332450ns) {HandleHandshakeStatus:329(98959780ns) {DropIncomingHandshake:441(15232ns) DropOutgoingHandshake:456(529ns) SwitchToState:178(449ns) ProcessPendingSessionEvents:495(29942143ns) {ForwardSessionEventToSession:612(18872151ns) ForwardSessionEventToSession:612(3681504ns) ForwardSessionEventToSession:612(5378ns) ForwardSessionEventToSession:612(3308ns) ForwardSessionEventToSession:612(950492ns) ForwardSessionEventToSession:612(1405909ns) ForwardSessionEventToSession:612(5987ns) ForwardSessionEventToSession:612(2619780ns) ForwardSessionEventToSession:612(2894ns) ForwardSessionEventToSession:612(2372183ns)}}}
2025-11-11T17:21:22.619517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:22.619647Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519171600408556:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:22.624318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519171600408533:2107] [node 4] ICP22 created new session: [1:7571519171600408571:2107]
2025-11-11T17:21:22.629874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519171600408571:2107] [node 4] ICS09 handshake done sender: [1:7571519171600408552:2118] self: [1:7571519171600408549:0] peer: [4:7571519172145818157:0] socket: 670 qp: -1
2025-11-11T17:21:22.631315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519171600408536:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.631386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:22.631445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519171600408536:2110] [node 6] ICP22 created new session: [1:7571519171600408572:2110]
2025-11-11T17:21:22.631489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519171600408572:2110] [node 6] ICS09 handshake done sender: [1:7571519171600408556:2119] self: [1:7571519171600408555:0] peer: [6:7571519172108261240:0] socket: 626 qp: -1
2025-11-11T17:21:22.631531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519171600408572:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:22.631563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:22.632486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519171600408571:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:22.632520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:22.482913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519167813293852:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(48981869ns) {HandleHandshakeStatus:329(48970657ns) {DropIncomingHandshake:441(552ns) DropOutgoingHandshake:456(6024ns) SwitchToState:178(585ns) ProcessPendingSessionEvents:495(3671445ns) {ForwardSessionEventToSession:612(3606382ns) ForwardSessionEventToSession:612(13288ns) ForwardSessionEventToSession:612(6760ns) ForwardSessionEventToSession:612(4042ns) ForwardSessionEventToSession:612(5157ns) ForwardSessionEventToSession:612(5671ns) ForwardSessionEventToSession:612(3177ns) ForwardSessionEventToSession:612(8200ns) ForwardSessionEventToSession:612(5546ns) ForwardSessionEventToSession:612(2190ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.481369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519172145818135:2108] [node 5] ICP22 created new session: [4:7571519172145818151:2108]
2025-11-11T17:21:22.559874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519172649449891:2049] [node 2] ICS09 handshake done sender: [5:7571519172649449871:2117] self: [5:7571519172649449888:0] peer: [2:7571519173781728756:0] socket: 660 qp: -1
2025-11-11T17:21:22.475269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:22.623989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519173781728745:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(107445886ns) {HandleHandshakeStatus:329(96571729ns) {DropIncomingHandshake:441(3090716ns) DropOutgoingHandshake:456(483ns) SwitchToState:178(886ns) ProcessPendingSessionEvents:495(33112356ns) {ForwardSessionEventToSession:612(8098265ns) ForwardSessionEventToSession:612(4837910ns) ForwardSessionEventToSession:612(7311ns) ForwardSessionEventToSession:612(4250ns) ForwardSessionEventToSession:612(17640496ns) ForwardSessionEventToSession:612(10892ns) ForwardSessionEventToSession:612(3658ns) ForwardSessionEventToSession:612(1328011ns) ForwardSessionEventToSession:612(3573ns) ForwardSessionEventToSession:612(1154646ns)}}}
2025-11-11T17:21:22.495229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261232:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.559908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519172649449891:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:22.559941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:22.575268Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519172649449873:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:22.580607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519172649449858:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.586024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:22.590230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519172649449858:2109] [node 6] ICP22 created new session: [5:7571519172649449894:2109]
2025-11-11T17:21:22.596260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519172649449894:2109] [node 6] ICS09 handshake done sender: [5:7571519172649449873:2119] self: [5:7571519172649449869:0] peer: [6:7571519172108261239:0] socket: 662 qp: -1
2025-11-11T17:21:22.606161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519172649449894:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:22.475317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519170351115373:2048] [node 1] ICP22 created new session: [3:7571519174646082760:2048]
2025-11-11T17:21:22.678162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519171600408533:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(45935612ns) {HandleHandshakeStatus:329(45933257ns) {DropIncomingHandshake:441(16730ns) DropOutgoingHandshake:456(275ns) SwitchToState:178(399ns) ProcessPendingSessionEvents:495(6997490ns) {ForwardSessionEventToSession:612(3940749ns) ForwardSessionEventToSession:612(8726ns) ForwardSessionEventToSession:612(3245ns) ForwardSessionEventToSession:612(5816ns) ForwardSessionEventToSession:612(4583ns) ForwardSessionEventToSession:612(2805ns) ForwardSessionEventToSession:612(673059ns) ForwardSessionEventToSession:612(3924ns) ForwardSessionEventToSession:612(1349976ns) ForwardSessionEventToSession:612(990864ns)}}}
2025-11-11T17:21:22.493087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519172145818153:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7185
2025-11-11T17:21:22.606219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:22.667004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519172649449858:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(74939997ns) {HandleHandshakeStatus:329(74507434ns) {DropIncomingHandshake:441(16943ns) DropOutgoingHandshake:456(6122ns) SwitchToState:178(659ns) ProcessPendingSessionEvents:495(28892221ns) {ForwardSessionEventToSession:612(8041352ns) ForwardSessionEventToSession:612(6229ns) ForwardSessionEventToSession:612(825760ns) ForwardSessionEventToSession:612(4234ns) ForwardSessionEventToSession:612(20663ns) ForwardSessionEventToSession:612(4312816ns) ForwardSessionEventToSession:612(3689ns) ForwardSessionEventToSession:612(3753547ns) ForwardSessionEventToSession:612(810048ns) ForwardSessionEventToSession:612(7292ns)}}}
2025-11-11T17:21:22.512217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261237:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.637822Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728755:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.496982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519172145818151:2108] [node 5] ICS09 handshake done sender: [4:7571519172145818141:2114] self: [4:7571519172145818139:0] peer: [5:7571519172649449876:0] socket: 659 qp: -1
2025-11-11T17:21:22.475367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519174646082760:2048] [node 1] ICS09 handshake done sender: [3:7571519174646082750:2117] self: [3:7571519174646082754:0] peer: [1:7571519171600408537:0] socket: 651 qp: -1
2025-11-11T17:21:22.498756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519172145818151:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:22.512724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:22.519644Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519172145818153:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.519751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519167850850771:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.522244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519172145818149:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7185
2025-11-11T17:21:22.550599Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519172145818149:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.552075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519172145818156:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7185
2025-11-11T17:21:22.555383Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.565988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519167850850769:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.577463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519167850850771:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519172145818153:2117] held: no
2025-11-11T17:21:22.590571Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519172145818153:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:22.601671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519167850850771:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.605910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:22.611282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519172145818135:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(114928808ns) {HandleHandshakeStatus:329(114923714ns) {DropIncomingHandshake:441(20816ns) DropOutgoingHandshake:456(556ns) SwitchToState:178(814ns) ProcessPendingSessionEvents:495(36180814ns) {ForwardSessionEventToSession:612(20016031ns) ForwardSessionEventToSession:612(5845ns) ForwardSessionEventToSession:612(7464180ns) ForwardSessionEventToSession:612(3126ns) ForwardSessionEventToSession:612(768593ns) ForwardSessionEventToSession:612(4446ns) ForwardSessionEventToSession:612(3713ns) ForwardSessionEventToSession:612(4498675ns) ForwardSessionEventToSession:612(4327ns) ForwardSessionEventToSession:612(2573ns)}}}
2025-11-11T17:21:22.619423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519167850850771:2050] [node 3] ICP22 created new session: [4:7571519172145818160:2050]
2025-11-11T17:21:22.624276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519172145818160:2050] [node 3] ICS09 handshake done sender: [4:7571519172145818153:2117] self: [4:7571519172145818155:0] peer: [3:7571519174646082748:0] socket: 639 qp: -1
2025-11-11T17:21:22.626298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519172145818160:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:22.626733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:22.632925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519172145818158:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7185
2025-11-11T17:21:22.650560Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.650787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519167850850771:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(39220274ns) {HandleHandshakeStatus:329(39218100ns) {DropIncomingHandshake:441(3170ns) DropOutgoingHandshake:456(417ns) SwitchToState:178(466ns) ProcessPendingSessionEvents:495(860ns)}}
2025-11-11T17:21:22.651461Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519172145818158:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.655092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519167850850769:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519172145818149:2116] held: no
2025-11-11T17:21:22.662604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519167850850770:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.667116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519172145818159:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7185
2025-11-11T17:21:22.671564Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519172145818149:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:22.672273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519167850850769:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.672314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:22.672363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519167850850769:2048] [node 1] ICP22 created new session: [4:7571519172145818163:2048]
2025-11-11T17:21:22.672410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519172145818163:2048] [node 1] ICS09 handshake done sender: [4:7571519172145818149:2116] self: [4:7571519172145818157:0] peer: [1:7571519171600408549:0] socket: 671 qp: -1
2025-11-11T17:21:22.672444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519172145818163:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:22.672485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:22.519117Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519172108261232:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.475417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519174646082760:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:22.475447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:22.519963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082749:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.637850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519173781728755:2124] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.519986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519174646082749:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.521085Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519174646082749:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:22.521331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519174646082732:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.521387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:22.521429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519174646082732:2107] [node 4] ICP22 created new session: [3:7571519174646082764:2107]
2025-11-11T17:21:22.521470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519174646082764:2107] [node 4] ICS09 handshake done sender: [3:7571519174646082749:2116] self: [3:7571519174646082748:0] peer: [4:7571519172145818155:0] socket: 627 qp: -1
2025-11-11T17:21:22.521500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519174646082764:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:22.521534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:22.542041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519174646082732:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(20501583ns) {HandleHandshakeStatus:329(20499188ns) {DropIncomingHandshake:441(12405ns) DropOutgoingHandshake:456(460ns) SwitchToState:178(227ns) ProcessPendingSessionEvents:495(8064500ns) {ForwardSessionEventToSession:612(8011248ns) ForwardSessionEventToSession:612(5901ns) ForwardSessionEventToSession:612(10728ns) ForwardSessionEventToSession:612(3453ns) ForwardSessionEventToSession:612(2796ns) ForwardSessionEventToSession:612(4761ns) ForwardSessionEventToSession:612(3055ns) ForwardSessionEventToSession:612(3347ns) ForwardSessionEventToSession:612(5040ns) ForwardSessionEventToSession:612(3087ns)}}}
2025-11-11T17:21:22.574299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519174646082757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60452
2025-11-11T17:21:22.583241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082757:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.583326Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082745:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.583352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519174646082745:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.584596Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519174646082745:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:22.584763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519174646082733:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.584824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:22.584904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519174646082733:2109] [node 6] ICP22 created new session: [3:7571519174646082769:2109]
2025-11-11T17:21:22.584974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519174646082769:2109] [node 6] ICS09 handshake done sender: [3:7571519174646082745:2114] self: [3:7571519174646082744:0] peer: [6:7571519172108261241:0] socket: 680 qp: -1
2025-11-11T17:21:22.585020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519174646082769:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:22.585054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:22.601930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519174646082766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60452
2025-11-11T17:21:22.629790Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519174646082766:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.633830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519170351115374:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.639291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519174646082772:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60452
2025-11-11T17:21:22.640006Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519174646082766:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:22.640262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519170351115374:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.640340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:22.640387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519170351115374:2049] [node 2] ICP22 created new session: [3:7571519174646082773:2049]
2025-11-11T17:21:22.640432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519174646082773:2049] [node 2] ICS09 handshake done sender: [3:7571519174646082766:2120] self: [3:7571519174646082771:0] peer: [2:7571519173781728754:0] socket: 644 qp: -1
2025-11-11T17:21:22.640464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519174646082773:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:22.640501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:22.519302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519172108261208:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.530164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261238:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.534637Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519172108261237:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.565764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519167813293851:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.566243Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519172108261238:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.575626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519167813293853:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:22.595124Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.602104Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.610207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519172108261208:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519172108261232:2118] held: no
2025-11-11T17:21:22.610342Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519167813293853:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519172108261238:2121] held: no
2025-11-11T17:21:22.610846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261243:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.613903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261244:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.618011Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519172108261238:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:22.618255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519167813293853:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.618312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:22.618365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519167813293853:2050] [node 3] ICP22 created new session: [6:7571519172108261245:2050]
2025-11-11T17:21:22.618411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519172108261245:2050] [node 3] ICS09 handshake done sender: [6:7571519172108261238:2121] self: [6:7571519172108261241:0] peer: [3:7571519174646082744:0] socket: 632 qp: -1
2025-11-11T17:21:22.618460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519172108261245:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:22.618495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:22.634093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519172108261237:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:22.651029Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.654471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519167813293851:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.654533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:22.655149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519172108261242:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45152
2025-11-11T17:21:22.655274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519167813293851:2048] [node 1] ICP22 created new session: [6:7571519172108261247:2048]
2025-11-11T17:21:22.657801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519172108261247:2048] [node 1] ICS09 handshake done sender: [6:7571519172108261237:2120] self: [6:7571519172108261240:0] peer: [1:7571519171600408555:0] socket: 648 qp: -1
2025-11-11T17:21:22.657846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519172108261247:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:22.657894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:22.658126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519167813293851:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:22.664826Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519172108261232:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:22.670666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519172108261208:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.677940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:22.679181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519172108261208:2105] [node 5] ICP22 created new session: [6:7571519172108261249:2105]
2025-11-11T17:21:22.679237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519172108261249:2105] [node 5] ICS09 handshake done sender: [6:7571519172108261232:2118] self: [6:7571519172108261239:0] peer: [5:7571519172649449869:0] socket: 663 qp: -1
2025-11-11T17:21:22.679272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519172108261249:2105] [node 5] ICS10 traffic start
2025-11-11T17:21:22.679334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:22.639535Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519173781728755:2124] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:22.642386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519169486761375:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.642445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:22.642491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519169486761375:2049] [node 3] ICP22 created new session: [2:7571519173781728781:2049]
2025-11-11T17:21:22.642534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519173781728781:2049] [node 3] ICS09 handshake done sender: [2:7571519173781728755:2124] self: [2:7571519173781728754:0] peer: [3:7571519174646082771:0] socket: 647 qp: -1
2025-11-11T17:21:22.642567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519173781728781:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:22.642606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:22.654115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519173781728743:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(44454795ns) {HandleHandshakeStatus:329(44452359ns) {DropIncomingHandshake:441(33273ns) DropOutgoingHandshake:456(447ns) SwitchToState:178(706ns) ProcessPendingSessionEvents:495(16347459ns) {ForwardSessionEventToSession:612(7703223ns) ForwardSessionEventToSession:612(22368ns) ForwardSessionEventToSession:612(1940529ns) ForwardSessionEventToSession:612(4862ns) ForwardSessionEventToSession:612(3518ns) ForwardSessionEventToSession:612(280250ns) ForwardSessionEventToSession:612(4357ns) ForwardSessionEventToSession:612(2210577ns) ForwardSessionEventToSession:612(4133398ns) ForwardSessionEventToSession:612(13867ns)}}}
2025-11-11T17:21:22.678730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519173781728763:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:22.678767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519173781728763:2127] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:22.693836Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519173781728763:2127] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:22.694206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519173781728744:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:22.694280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:22.694333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519173781728744:2107] [node 4] ICP22 created new session: [2:7571519173781728784:2107]
2025-11-11T17:21:22.694379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519173781728784:2107] [node 4] ICS09 handshake done sender: [2:7571519173781728763:2127] self: [2:7571519173781728762:0] peer: [4:7571519172145818162:0] socket: 684 qp: -1
2025-11-11T17:21:22.694410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519173781728784:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:22.694449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:22.695818Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25672) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:22.696662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519167850850770:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25672) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519172145818158:2119] held: no
2025-11-11T17:21:22.696882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519172145818165:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7185
2025-11-11T17:21:22.697351Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519172145818158:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:22.697603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519167850850770:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:22.697676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:22.697719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519167850850770:2049] [node 2] ICP22 created new session: [4:7571519172145818166:2049]
2025-11-11T17:21:22.697763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519172145818166:2049] [node 2] ICS09 handshake done sender: [4:7571519172145818158:2119] self: [4:7571519172145818162:0] peer: [2:7571519173781728762:0] socket: 686 qp: -1
2025-11-11T17:21:22.697801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519172145818166:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:22.697838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:22.775615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:22.775800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519173781728771:2048] [node 1] ICS01 socket: 654 reason# EndOfStream
2025-11-11T17:21:22.775830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519169486761374:2048] [node 1] ICP30 unregister session Session# [2:7571519173781728771:2048] VirtualId# [2:7571519173781728765:0]
2025-11-11T17:21:22.775858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519173781728771:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.775918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:22.775962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:22.779898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:22.780129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519174646082760:2048] [node 1] ICS01 socket: 651 reason# EndOfStream
2025-11-11T17:21:22.780167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519170351115373:2048] [node 1] ICP30 unregister session Session# [3:7571519174646082760:2048] VirtualId# [3:7571519174646082754:0]
2025-11-11T17:21:22.780202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519174646082760:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.780288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:22.780336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:22.781716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:22.781913Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:22.782037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519172649449886:2048] [node 1] ICS01 socket: 658 reason# EndOfStream
2025-11-11T17:21:22.782060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519168354482495:2048] [node 1] ICP30 unregister session Session# [5:7571519172649449886:2048] VirtualId# [5:7571519172649449877:0]
2025-11-11T17:21:22.782081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519172649449886:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.782134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:22.782159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:22.781958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519172145818163:2048] [node 1] ICS01 socket: 671 reason# EndOfStream
2025-11-11T17:21:22.781996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519167850850769:2048] [node 1] ICP30 unregister session Session# [4:7571519172145818163:2048] VirtualId# [4:7571519172145818157:0]
2025-11-11T17:21:22.782028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519172145818163:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.782104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:22.782147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:22.783850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:22.784334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519172108261247:2048] [node 1] ICS01 socket: 648 reason# EndOfStream
2025-11-11T17:21:22.784386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519167813293851:2048] [node 1] ICP30 unregister session Session# [6:7571519172108261247:2048] VirtualId# [6:7571519172108261240:0]
2025-11-11T17:21:22.784421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519172108261247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.784487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:22.784529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:22.853276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:22.853491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519174646082773:2049] [node 2] ICS01 socket: 644 reason# EndOfStream
2025-11-11T17:21:22.853535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519170351115374:2049] [node 2] ICP30 unregister session Session# [3:7571519174646082773:2049] VirtualId# [3:7571519174646082771:0]
2025-11-11T17:21:22.853568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519174646082773:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.853659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:22.853699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:22.856528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:22.856721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519172145818166:2049] [node 2] ICS01 socket: 686 reason# EndOfStream
2025-11-11T17:21:22.856754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519167850850770:2049] [node 2] ICP30 unregister session Session# [4:7571519172145818166:2049] VirtualId# [4:7571519172145818162:0]
2025-11-11T17:21:22.856797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519172145818166:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.856876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:22.856913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:22.857265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:22.857420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519172649449891:2049] [node 2] ICS01 socket: 660 reason# EndOfStream
2025-11-11T17:21:22.857452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519168354482496:2049] [node 2] ICP30 unregister session Session# [5:7571519172649449891:2049] VirtualId# [5:7571519172649449888:0]
2025-11-11T17:21:22.857476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519172649449891:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.857529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:22.857562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:22.861745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:22.861983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519172108261235:2049] [node 2] ICS01 socket: 636 reason# EndOfStream
2025-11-11T17:21:22.862032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519167813293852:2049] [node 2] ICP30 unregister session Session# [6:7571519172108261235:2049] VirtualId# [6:7571519172108261229:0]
2025-11-11T17:21:22.862070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519172108261235:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.862161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:22.862218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:22.925344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:22.925663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519172145818160:2050] [node 3] ICS01 socket: 639 reason# EndOfStream
2025-11-11T17:21:22.925710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519167850850771:2050] [node 3] ICP30 unregister session Session# [4:7571519172145818160:2050] VirtualId# [4:7571519172145818155:0]
2025-11-11T17:21:22.925783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519172145818160:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.925891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:22.925940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:22.927310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:22.927594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519172108261245:2050] [node 3] ICS01 socket: 632 reason# EndOfStream
2025-11-11T17:21:22.927680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519167813293853:2050] [node 3] ICP30 unregister session Session# [6:7571519172108261245:2050] VirtualId# [6:7571519172108261241:0]
2025-11-11T17:21:22.927721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519172108261245:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.927820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:22.927885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:22.926357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:22.926518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519172649449883:2050] [node 3] ICS01 socket: 640 reason# EndOfStream
2025-11-11T17:21:22.926553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519168354482497:2050] [node 3] ICP30 unregister session Session# [5:7571519172649449883:2050] VirtualId# [5:7571519172649449868:0]
2025-11-11T17:21:22.926580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519172649449883:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:22.926642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:22.926718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:23.015171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:23.015437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519172108261230:2109] [node 4] ICS01 socket: 635 reason# EndOfStream
2025-11-11T17:21:23.015480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519172108261211:2109] [node 4] ICP30 unregister session Session# [6:7571519172108261230:2109] VirtualId# [6:7571519172108261227:0]
2025-11-11T17:21:23.015513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519172108261230:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:23.015627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:23.015686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:23.017302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:23.017629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519172649449881:2108] [node 4] ICS01 socket: 661 reason# EndOfStream
2025-11-11T17:21:23.017698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519172649449854:2108] [node 4] ICP30 unregister session Session# [5:7571519172649449881:2108] VirtualId# [5:7571519172649449876:0]
2025-11-11T17:21:23.017740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519172649449881:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:23.017844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:23.017881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:23.049905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:23.050153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519172108261249:2105] [node 5] ICS01 socket: 663 reason# EndOfStream
2025-11-11T17:21:23.050200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519172108261208:2105] [node 5] ICP30 unregister session Session# [6:7571519172108261249:2105] VirtualId# [6:7571519172108261239:0]
2025-11-11T17:21:23.050239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519172108261249:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:23.050346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:23.050396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:23.198199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519175852343127:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:23.199085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519175852343126:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:23.216209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519178179160059:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:23.216255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519178179160060:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:23.237870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519176455591564:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:23.237926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519176455591565:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:23.263309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519175612303661:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:23.263351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519175612303662:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:23.263380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519175612303663:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:23.288927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519175823423099:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:23.289070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519175823423101:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:23.289211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519175823423100:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:23.308845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519177724465085:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:23.308871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519177724465086:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:23.309276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519177724465084:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:23.538215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519177724465142:2099] [node 5] ICP01 ready to work
2025-11-11T17:21:23.540378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519175852343187:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:23.599394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519175852343190:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.646605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519175823423160:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:23.603193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519177724465142:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(64628453ns) {Configure:104(64624469ns) {ProcessConfigured:135(96752ns) {StartInitialHandshake:157(93540ns) {DropHandshakes:469(5518ns) {DropIncomingHandshake:441(714ns) DropOutgoingHandshake:456(350ns)} GenerateSessionVirtualId:988(1059ns) PrepareNewSessionHandshake:476(1556ns) {SwitchToState:178(456ns)}}}}}
2025-11-11T17:21:23.619842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519176455591625:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:23.638126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519178179160059:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53673889ns) {Configure:104(53663957ns) {ProcessConfigured:135(63838ns) {StartInitialHandshake:157(60573ns) {DropHandshakes:469(2933ns) {DropIncomingHandshake:441(574ns) DropOutgoingHandshake:456(485ns)} GenerateSessionVirtualId:988(679ns) PrepareNewSessionHandshake:476(1751ns) {SwitchToState:178(537ns)}}}}}
2025-11-11T17:21:23.624639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519175852343191:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:23.655885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519176455591565:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(45769304ns) {Configure:104(45762934ns) {ProcessConfigured:135(12604528ns) {StartInitialHandshake:157(12600659ns) {DropHandshakes:469(3901ns) {DropIncomingHandshake:441(790ns) DropOutgoingHandshake:456(1060ns)} GenerateSessionVirtualId:988(1568ns) PrepareNewSessionHandshake:476(4756ns) {SwitchToState:178(2404ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.641610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519178179160122:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:23.686452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519175612303723:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:23.647465Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519175852343187:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(106764899ns) {Configure:104(106761481ns) {ProcessConfigured:135(12882187ns) {StartInitialHandshake:157(12878998ns) {DropHandshakes:469(3031ns) {DropIncomingHandshake:441(515ns) DropOutgoingHandshake:456(590ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:476(5308ns) {SwitchToState:178(2266ns)}}}}}
2025-11-11T17:21:23.709976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519177724465149:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.669383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519176455591631:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:23.702315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519175612303725:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:23.648545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519178179160126:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:23.694303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519175852343127:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(160394248ns) {Configure:104(160390527ns) {ProcessConfigured:135(25817671ns) {StartInitialHandshake:157(25814586ns) {DropHandshakes:469(3067ns) {DropIncomingHandshake:441(543ns) DropOutgoingHandshake:456(575ns)} GenerateSessionVirtualId:988(1467ns) PrepareNewSessionHandshake:476(6440ns) {SwitchToState:178(4297ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.717665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519176455591632:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:23.794805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519177724465086:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(125216151ns) {Configure:104(125209189ns) {ProcessConfigured:135(83967ns) {StartInitialHandshake:157(80988ns) {DropHandshakes:469(2646ns) {DropIncomingHandshake:441(504ns) DropOutgoingHandshake:456(385ns)} GenerateSessionVirtualId:988(967ns) PrepareNewSessionHandshake:476(1656ns) {SwitchToState:178(566ns)}}}}}
2025-11-11T17:21:23.803805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519175612303729:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15413
2025-11-11T17:21:23.676188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519178179160127:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:23.872392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519176455591632:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(75198996ns) {Configure:104(75189194ns) {ProcessConfigured:135(73389ns) {StartInitialHandshake:157(70868ns) {DropHandshakes:469(3627ns) {DropIncomingHandshake:441(1116ns) DropOutgoingHandshake:456(679ns)} GenerateSessionVirtualId:988(1094ns) PrepareNewSessionHandshake:476(2341ns) {SwitchToState:178(649ns)}}}}}
2025-11-11T17:21:23.805053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519175852343197:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37433
2025-11-11T17:21:23.719186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519178179160128:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36949
2025-11-11T17:21:23.828226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519178179160122:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(152686636ns) {Configure:104(152676275ns) {ProcessConfigured:135(68819ns) {StartInitialHandshake:157(65997ns) {DropHandshakes:469(2797ns) {DropIncomingHandshake:441(593ns) DropOutgoingHandshake:456(513ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:476(1522ns) {SwitchToState:178(634ns)}}}}}
2025-11-11T17:21:23.846031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519178179160132:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36949
2025-11-11T17:21:23.799682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519177724465153:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:23.849630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519175823423163:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:23.842263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519177724465085:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26356045ns) {Configure:104(26351671ns) {ProcessConfigured:135(63499ns) {StartInitialHandshake:157(60430ns) {DropHandshakes:469(2792ns) {DropIncomingHandshake:441(775ns) DropOutgoingHandshake:456(467ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:476(1632ns) {SwitchToState:178(624ns)}}}}}
2025-11-11T17:21:23.849648Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343197:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.854050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519175612303723:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(167279847ns) {Configure:104(167276697ns) {ProcessConfigured:135(72296ns) {StartInitialHandshake:157(69179ns) {DropHandshakes:469(2808ns) {DropIncomingHandshake:441(893ns) DropOutgoingHandshake:456(437ns)} GenerateSessionVirtualId:988(1046ns) PrepareNewSessionHandshake:476(2014ns) {SwitchToState:178(783ns)}}}}}
2025-11-11T17:21:23.890005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519175852343126:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(357004231ns) {Configure:104(356997964ns) {ProcessConfigured:135(19993694ns) {StartInitialHandshake:157(19989862ns) {DropHandshakes:469(2413ns) {DropIncomingHandshake:441(569ns) DropOutgoingHandshake:456(514ns)} GenerateSessionVirtualId:988(1875ns) PrepareNewSessionHandshake:476(7460ns) {SwitchToState:178(2426ns)}}}}}
2025-11-11T17:21:23.907431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519175852343201:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37433
2025-11-11T17:21:23.893660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519175823423166:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:23.923004Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519175823423160:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.913982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519178179160060:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(288085938ns) {Configure:104(288076037ns) {ProcessConfigured:135(78737ns) {StartInitialHandshake:157(76365ns) {DropHandshakes:469(2832ns) {DropIncomingHandshake:441(651ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:476(1483ns) {SwitchToState:178(629ns)}}}}}
2025-11-11T17:21:23.914823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519177724465153:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.946247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519175823423100:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(226349375ns) {Configure:104(226344783ns) {ProcessConfigured:135(12233166ns) {StartInitialHandshake:157(12229668ns) {DropHandshakes:469(3165ns) {DropIncomingHandshake:441(723ns) DropOutgoingHandshake:456(631ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:476(3561ns) {SwitchToState:178(1574ns)}}}}}
2025-11-11T17:21:23.932000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343201:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.947427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519176455591636:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32332
2025-11-11T17:21:23.947499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519176455591638:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32332
2025-11-11T17:21:23.947862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519176455591564:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(348501158ns) {Configure:104(348496409ns) {ProcessConfigured:135(13747008ns) {StartInitialHandshake:157(13744800ns) {DropHandshakes:469(762ns) {DropIncomingHandshake:441(139ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(2344ns) {SwitchToState:178(485ns)}}}}}
2025-11-11T17:21:23.948121Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519176455591638:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.948305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519176455591565:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:23.937925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519175612303662:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(203734961ns) {Configure:104(203727142ns) {ProcessConfigured:135(12150605ns) {StartInitialHandshake:157(12147599ns) {DropHandshakes:469(2668ns) {DropIncomingHandshake:441(671ns) DropOutgoingHandshake:456(593ns)} GenerateSessionVirtualId:988(719ns) PrepareNewSessionHandshake:476(4773ns) {SwitchToState:178(2430ns)}}}}}
2025-11-11T17:21:23.932312Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160132:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.945918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519178179160136:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36949
2025-11-11T17:21:23.941653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519175612303729:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.966958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519175823423099:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(230890502ns) {Configure:104(230886318ns) {ProcessConfigured:135(26745175ns) {StartInitialHandshake:157(26742411ns) {DropHandshakes:469(2466ns) {DropIncomingHandshake:441(541ns) DropOutgoingHandshake:456(447ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:476(8840176ns) {SwitchToState:178(5847ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.969097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519176455591636:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.949233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519178179160137:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36949
2025-11-11T17:21:23.966992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519175852343190:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(321877312ns) {Configure:104(321869959ns) {ProcessConfigured:135(23053296ns) {StartInitialHandshake:157(15035144ns) {DropHandshakes:469(3216ns) {DropIncomingHandshake:441(351ns) DropOutgoingHandshake:456(552ns)} GenerateSessionVirtualId:988(1571ns) PrepareNewSessionHandshake:476(3074ns) {SwitchToState:178(1436ns)}}}}}
2025-11-11T17:21:23.967653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519175852343191:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(318839440ns) {Configure:104(318833598ns) {ProcessConfigured:135(7707680ns) {StartInitialHandshake:157(7704253ns) {DropHandshakes:469(2770ns) {DropIncomingHandshake:441(660ns) DropOutgoingHandshake:456(432ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:476(2587ns) {SwitchToState:178(850ns)}}}}}
2025-11-11T17:21:23.971219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519175852343206:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37433
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.971248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519175823423165:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:23.955159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160134:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.982411Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343199:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.982453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519175852343199:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:23.971694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519175823423164:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:23.978201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519175612303661:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(305094339ns) {Configure:104(305088250ns) {ProcessConfigured:135(13417775ns) {StartInitialHandshake:157(13415373ns) {DropHandshakes:469(3538ns) {DropIncomingHandshake:441(863ns) DropOutgoingHandshake:456(693ns)} GenerateSessionVirtualId:988(819ns) PrepareNewSessionHandshake:476(3399ns) {SwitchToState:178(1465ns)}}}}}
2025-11-11T17:21:23.987251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343193:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.955197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519178179160134:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:23.981718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519176455591564:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:23.982665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519176455591625:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(320786335ns) {Configure:104(320773696ns) {ProcessConfigured:135(6658746ns) {StartInitialHandshake:157(6656419ns) {DropHandshakes:469(389ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(137ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:476(3113ns) {SwitchToState:178(1406ns)}}}}}
2025-11-11T17:21:24.005445Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:23.987547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519177724465149:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(266642111ns) {Configure:104(266636354ns) {ProcessConfigured:135(7992988ns) {StartInitialHandshake:157(7990663ns) {DropHandshakes:469(1538ns) {DropIncomingHandshake:441(292ns) DropOutgoingHandshake:456(316ns)} GenerateSessionVirtualId:988(513ns) PrepareNewSessionHandshake:476(4012ns) {SwitchToState:178(1668ns)}}}}}
2025-11-11T17:21:23.978808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519175823423166:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32309851ns) {Configure:104(32305873ns) {ProcessConfigured:135(39859ns) {StartInitialHandshake:157(38670ns) {DropHandshakes:469(4320ns) {DropIncomingHandshake:441(190ns) DropOutgoingHandshake:456(470ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:476(492ns) {SwitchToState:178(116ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.980053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519175612303731:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.955422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160128:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.987297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519175852343193:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.016639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519176455591631:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(255519252ns) {Configure:104(233532346ns) {ProcessConfigured:135(5198373ns) {StartInitialHandshake:157(5196356ns) {DropHandshakes:469(1064ns) {DropIncomingHandshake:441(288ns) DropOutgoingHandshake:456(259ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:476(2274ns) {SwitchToState:178(911ns)}}}}}
2025-11-11T17:21:24.016848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519176455591565:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519176455591638:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.980078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519175612303731:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.010938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519177724465149:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:23.979018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519175823423165:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.966766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160136:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.997516Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519175852343199:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.979229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519175823423168:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:23.980976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519175612303739:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15413
2025-11-11T17:21:24.054012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519180750558945:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32332
2025-11-11T17:21:23.969659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519178179160059:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.030856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432460:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.000219Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343206:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.981256Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519175612303731:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:24.060463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519176455591565:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(25492067ns) {HandleHandshakeStatus:395(25486438ns) {LogHandshakeFail:472(17150ns) {UpdateErrorStateLog:544(6031ns)} DropOutgoingHandshake:456(1802ns) ProcessPendingSessionEvents:495(14383553ns) {DropSessionEvent:512(18075ns) DropSessionEvent:512(3946301ns) DropSessionEvent:512(31741ns) DropSessionEvent:512(11355ns) DropSessionEvent:512(5679ns) DropSessionEvent:512(10301260ns) DropSessionEvent:512(28391ns) DropSessionEvent:512(12655ns) DropSessionEvent:512(6635ns) DropSessionEvent:512(4846ns)}}}
2025-11-11T17:21:23.977022Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519178179160134:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:23.979470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519175823423166:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.006463Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343196:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.066364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432461:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.987007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519175823423168:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:23.981964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519175612303661:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:23.977091Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160137:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.062382Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519176455591634:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.006502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519175852343196:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.986417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519175612303723:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:23.998471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519175823423163:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(124220131ns) {Configure:104(124215091ns) {ProcessConfigured:135(10111766ns) {StartInitialHandshake:157(10110083ns) {DropHandshakes:469(4950738ns) {DropIncomingHandshake:441(1162ns) DropOutgoingHandshake:456(777ns)} GenerateSessionVirtualId:988(636ns) PrepareNewSessionHandshake:476(1587ns) {SwitchToState:178(107ns)}}}}}
2025-11-11T17:21:24.062426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519176455591634:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.109656Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519177724465153:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:23.985677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519178179160126:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(268053707ns) {Configure:104(268048175ns) {ProcessConfigured:135(6369070ns) {StartInitialHandshake:157(6367268ns) {DropHandshakes:469(1448ns) {DropIncomingHandshake:441(336ns) DropOutgoingHandshake:456(326ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:476(2894ns) {SwitchToState:178(1184ns)}}}}}
2025-11-11T17:21:24.050874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519175852343193:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:23.987570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:24.110206Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519182019432460:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.020687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519175823423101:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(138792260ns) {Configure:104(138786489ns) {ProcessConfigured:135(5618657ns) {StartInitialHandshake:157(5617280ns) {DropHandshakes:469(3678046ns) {DropIncomingHandshake:441(326ns) DropOutgoingHandshake:456(1250ns)} GenerateSessionVirtualId:988(1889ns) PrepareNewSessionHandshake:476(2219ns) {SwitchToState:178(445ns)}}}}}
2025-11-11T17:21:24.084516Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519176455591636:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:23.986415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519178179160127:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(246667833ns) {Configure:104(246661212ns) {ProcessConfigured:135(8244627ns) {StartInitialHandshake:157(8242448ns) {DropHandshakes:469(1218ns) {DropIncomingHandshake:441(209ns) DropOutgoingHandshake:456(245ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:476(2708ns) {SwitchToState:178(754ns)}}}}}
2025-11-11T17:21:24.061544Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519175852343196:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:23.991158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519175612303723:2108] [node 5] ICP22 created new session: [4:7571519175612303742:2108]
2025-11-11T17:21:24.042341Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519175823423164:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.110419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519177724465085:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.095890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519176455591646:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32332
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.004555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519178179160060:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.046045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519175823423178:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:23.996136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519175612303742:2108] [node 5] ICS09 handshake done sender: [4:7571519175612303731:2112] self: [4:7571519175612303730:0] peer: [5:7571519175823423174:0] socket: 646 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.004643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:24.125376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519177724465149:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.100728Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519176455591634:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:24.061747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519175852343126:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:23.996202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519175612303742:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:23.996274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:24.010520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519175612303663:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(188901003ns) {Configure:104(188893273ns) {ProcessConfigured:135(4938464ns) {StartInitialHandshake:157(4936993ns) {DropHandshakes:469(2062ns) {DropIncomingHandshake:441(504ns) DropOutgoingHandshake:456(393ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:476(1338ns) {SwitchToState:178(124ns)}}}}}
2025-11-11T17:21:24.030024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519179907271042:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15413
2025-11-11T17:21:24.040665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519175612303727:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.040713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519175612303727:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.049998Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519175612303727:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:24.055530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519179907271043:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15413
2025-11-11T17:21:24.066022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519175612303725:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.066095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:24.074379Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519179907271043:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.080204Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519175612303729:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:24.087415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519175612303725:2109] [node 6] ICP22 created new session: [4:7571519179907271045:2109]
2025-11-11T17:21:24.092262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519175612303661:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.092336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:24.092386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519175612303661:2048] [node 1] ICP22 created new session: [4:7571519179907271047:2048]
2025-11-11T17:21:24.092434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519179907271047:2048] [node 1] ICS09 handshake done sender: [4:7571519175612303729:2111] self: [4:7571519175612303738:0] peer: [1:7571519175852343192:0] socket: 637 qp: -1
2025-11-11T17:21:24.092470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519179907271047:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:24.092502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:24.092987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519175612303663:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.101753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519179907271050:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15413
2025-11-11T17:21:24.109537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519179907271045:2109] [node 6] ICS09 handshake done sender: [4:7571519175612303727:2110] self: [4:7571519175612303726:0] peer: [6:7571519182019432459:0] socket: 635 qp: -1
2025-11-11T17:21:24.111815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519179907271045:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:24.114448Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519179907271050:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.114517Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519175612303739:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.114661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:24.115125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519175612303725:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(53199689ns) {HandleHandshakeStatus:329(53197464ns) {DropIncomingHandshake:441(1585ns) DropOutgoingHandshake:456(1228ns) SwitchToState:178(377ns) ProcessPendingSessionEvents:495(119012ns) {ForwardSessionEventToSession:612(62755ns) ForwardSessionEventToSession:612(7809ns) ForwardSessionEventToSession:612(2738ns) ForwardSessionEventToSession:612(2763ns) ForwardSessionEventToSession:612(5916ns) ForwardSessionEventToSession:612(2963ns) ForwardSessionEventToSession:612(5430ns) ForwardSessionEventToSession:612(7731ns) ForwardSessionEventToSession:612(2499ns) ForwardSessionEventToSession:612(4626ns)}}}
2025-11-11T17:21:24.125358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519175612303723:2108] [node 5] ICP18 handshake (actor [4:7571519175612303739:2115]) is too old
2025-11-11T17:21:24.125867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519175612303662:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.144921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519179907271053:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15413
2025-11-11T17:21:24.145306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519179907271054:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15413
2025-11-11T17:21:24.149766Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519179907271043:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:24.049848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519175823423100:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.139741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:24.065966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:24.102106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519180750558947:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32332
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.155275Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519179907271050:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:24.158868Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.010131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519178179160060:2049] [node 3] ICP22 created new session: [2:7571519182474127444:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.051657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519175823423101:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.078124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519175852343187:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.141156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519177724465149:2107] [node 4] ICP22 created new session: [6:7571519182019432465:2107]
2025-11-11T17:21:24.103101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519176455591564:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.026394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519182474127444:2049] [node 3] ICS09 handshake done sender: [2:7571519178179160134:2114] self: [2:7571519178179160133:0] peer: [3:7571519176455591641:0] socket: 651 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.098972Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519175823423165:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:24.103335Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.170448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519175612303663:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519179907271043:2118] held: no
2025-11-11T17:21:24.144417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519182019432465:2107] [node 4] ICS09 handshake done sender: [6:7571519177724465153:2112] self: [6:7571519182019432459:0] peer: [4:7571519175612303726:0] socket: 636 qp: -1
2025-11-11T17:21:24.078179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:24.178005Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.181908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519175612303662:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.181980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:24.182043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519175612303662:2049] [node 2] ICP22 created new session: [4:7571519179907271055:2049]
2025-11-11T17:21:24.182083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519179907271055:2049] [node 2] ICS09 handshake done sender: [4:7571519179907271050:2119] self: [4:7571519179907271051:0] peer: [2:7571519178179160142:0] socket: 683 qp: -1
2025-11-11T17:21:24.182122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519179907271055:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:24.182170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:24.182433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519175612303662:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:24.199058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519175612303663:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.199120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:24.111605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519175823423166:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.031430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519182474127444:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:24.078238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519175852343187:2107] [node 4] ICP22 created new session: [1:7571519180147310513:2107]
2025-11-11T17:21:24.145725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519182019432465:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:24.031490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519182474127443:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36949
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.106986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:24.213610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519175612303663:2050] [node 3] ICP22 created new session: [4:7571519179907271057:2050]
2025-11-11T17:21:24.117227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519180118390483:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:24.109570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519176455591564:2048] [node 1] ICP22 created new session: [3:7571519180750558948:2048]
2025-11-11T17:21:24.078286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519180147310513:2107] [node 4] ICS09 handshake done sender: [1:7571519175852343193:2110] self: [1:7571519175852343192:0] peer: [4:7571519175612303738:0] socket: 633 qp: -1
2025-11-11T17:21:24.047780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519182474127446:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36949
2025-11-11T17:21:24.111132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519180750558948:2048] [node 1] ICS09 handshake done sender: [3:7571519176455591636:2112] self: [3:7571519176455591644:0] peer: [1:7571519175852343195:0] socket: 650 qp: -1
2025-11-11T17:21:24.111201Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519176455591638:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:24.112334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519176455591632:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.112391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:24.112445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519176455591632:2108] [node 5] ICP22 created new session: [3:7571519180750558949:2108]
2025-11-11T17:21:24.112496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519180750558949:2108] [node 5] ICS09 handshake done sender: [3:7571519176455591634:2111] self: [3:7571519176455591633:0] peer: [5:7571519180118390479:0] socket: 648 qp: -1
2025-11-11T17:21:24.112523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519180750558949:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:24.112559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:24.119082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519180750558948:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:24.123123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519180750558947:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.123210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519176455591565:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.123256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:24.123303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519176455591565:2049] [node 2] ICP22 created new session: [3:7571519180750558951:2049]
2025-11-11T17:21:24.123349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519180750558951:2049] [node 2] ICS09 handshake done sender: [3:7571519176455591638:2113] self: [3:7571519176455591641:0] peer: [2:7571519178179160133:0] socket: 655 qp: -1
2025-11-11T17:21:24.125361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:24.133882Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519176455591643:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.133925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519176455591643:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.134299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519180750558951:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:24.135449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:24.137090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519180750558954:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32332
2025-11-11T17:21:24.140159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519176455591565:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16833332ns) {HandleHandshakeStatus:329(16830590ns) {DropIncomingHandshake:441(945ns) DropOutgoingHandshake:456(605ns) SwitchToState:178(865ns) ProcessPendingSessionEvents:495(1314ns)}}
2025-11-11T17:21:24.140253Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:24.142921Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519176455591643:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:24.151178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519176455591625:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.174066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519180750558958:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32332
2025-11-11T17:21:24.175604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:24.179375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519176455591625:2107] [node 4] ICP22 created new session: [3:7571519180750558959:2107]
2025-11-11T17:21:24.179637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519176455591564:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(48424223ns) {HandleHandshakeStatus:329(48422037ns) {DropIncomingHandshake:441(1849ns) DropOutgoingHandshake:456(20491ns) SwitchToState:178(1144ns) ProcessPendingSessionEvents:495(10155795ns) {ForwardSessionEventToSession:612(4870032ns) ForwardSessionEventToSession:612(5671ns) ForwardSessionEventToSession:612(1431556ns) ForwardSessionEventToSession:612(5491ns) ForwardSessionEventToSession:612(1397778ns) ForwardSessionEventToSession:612(633226ns) ForwardSessionEventToSession:612(3786ns) ForwardSessionEventToSession:612(1777105ns) ForwardSessionEventToSession:612(6796ns) ForwardSessionEventToSession:612(3364ns)}}}
2025-11-11T17:21:24.183869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519176455591564:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:24.199173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519180750558959:2107] [node 4] ICS09 handshake done sender: [3:7571519176455591643:2115] self: [3:7571519176455591642:0] peer: [4:7571519179907271049:0] socket: 634 qp: -1
2025-11-11T17:21:24.199225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519180750558958:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.202060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519180750558959:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:24.217984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:24.229657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519179907271057:2050] [node 3] ICS09 handshake done sender: [4:7571519179907271043:2118] self: [4:7571519179907271049:0] peer: [3:7571519176455591642:0] socket: 672 qp: -1
2025-11-11T17:21:24.145773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:24.117369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519180118390484:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.235789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519179907271057:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:24.054450Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160130:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.155998Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.078327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519180147310513:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:24.235864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:24.236158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519175612303663:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(37029832ns) {HandleHandshakeStatus:329(37024772ns) {DropIncomingHandshake:441(1106ns) DropOutgoingHandshake:456(914ns) SwitchToState:178(876ns) ProcessPendingSessionEvents:495(964ns)}}
2025-11-11T17:21:24.123600Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519175823423168:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:24.248343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519176455591625:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(96917631ns) {HandleHandshakeStatus:329(96914064ns) {DropIncomingHandshake:441(20970ns) DropOutgoingHandshake:456(1081ns) SwitchToState:178(817ns) ProcessPendingSessionEvents:495(8357569ns) {ForwardSessionEventToSession:612(8300223ns) ForwardSessionEventToSession:612(8563ns) ForwardSessionEventToSession:612(3218ns) ForwardSessionEventToSession:612(7097ns) ForwardSessionEventToSession:612(3056ns) ForwardSessionEventToSession:612(5706ns) ForwardSessionEventToSession:612(2964ns) ForwardSessionEventToSession:612(3056ns) ForwardSessionEventToSession:612(10663ns) ForwardSessionEventToSession:612(2691ns)}}}
2025-11-11T17:21:24.078366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:24.251943Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519176455591647:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.251967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519176455591647:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.261584Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519176455591647:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:24.162759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432464:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.054481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519178179160130:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.078971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519175852343127:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.124146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519175823423101:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.055163Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519178179160136:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:24.282936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519176455591631:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.162918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519177724465085:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519182019432460:2116] held: no
2025-11-11T17:21:24.079018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:24.079045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519175852343127:2049] [node 3] ICP22 created new session: [1:7571519180147310515:2049]
2025-11-11T17:21:24.124251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:24.169485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519177724465149:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(49501242ns) {HandleHandshakeStatus:329(49497818ns) {DropIncomingHandshake:441(2339ns) DropOutgoingHandshake:456(17672ns) SwitchToState:178(1124ns) ProcessPendingSessionEvents:495(11484096ns) {ForwardSessionEventToSession:612(11402694ns) ForwardSessionEventToSession:612(7178ns) ForwardSessionEventToSession:612(3724ns) ForwardSessionEventToSession:612(30875ns) ForwardSessionEventToSession:612(6702ns) ForwardSessionEventToSession:612(6029ns) ForwardSessionEventToSession:612(3036ns) ForwardSessionEventToSession:612(2544ns) ForwardSessionEventToSession:612(6166ns) ForwardSessionEventToSession:612(3142ns)}}}
2025-11-11T17:21:24.056061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519178179160059:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.124304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519175823423101:2050] [node 3] ICP22 created new session: [5:7571519180118390486:2050]
2025-11-11T17:21:24.079075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519180147310515:2049] [node 3] ICS09 handshake done sender: [1:7571519175852343196:2111] self: [1:7571519175852343195:0] peer: [3:7571519176455591644:0] socket: 649 qp: -1
2025-11-11T17:21:24.175166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432463:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.079101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519180147310515:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:24.056123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:24.300821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:24.124347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519180118390486:2050] [node 3] ICS09 handshake done sender: [5:7571519175823423168:2113] self: [5:7571519180118390479:0] peer: [3:7571519176455591633:0] socket: 654 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:24.305098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519176455591631:2109] [node 6] ICP22 created new session: [3:7571519180750558963:2109]
2025-11-11T17:21:24.179437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519182019432464:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.184573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519177724465142:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.187889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432471:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.188479Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519182019432464:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:24.188806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519177724465142:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.188880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:24.188944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519177724465142:2099] [node 5] ICP22 created new session: [6:7571519182019432472:2099]
2025-11-11T17:21:24.189002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519182019432472:2099] [node 5] ICS09 handshake done sender: [6:7571519182019432464:2119] self: [6:7571519182019432470:0] peer: [5:7571519175823423176:0] socket: 674 qp: -1
2025-11-11T17:21:24.189054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519182019432472:2099] [node 5] ICS10 traffic start
2025-11-11T17:21:24.189091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:24.202322Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519182019432463:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.233179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519177724465084:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.239220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432468:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.249782Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519182019432468:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.249927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519177724465086:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.258130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432469:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.288285Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519182019432460:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:24.288516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432477:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.297475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519177724465085:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.299239Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519182019432468:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:24.300908Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.304171Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.310060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:24.315984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519177724465086:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.316087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:24.316145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519177724465086:2050] [node 3] ICP22 created new session: [6:7571519182019432479:2050]
2025-11-11T17:21:24.316203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519182019432479:2050] [node 3] ICS09 handshake done sender: [6:7571519182019432468:2120] self: [6:7571519182019432475:0] peer: [3:7571519176455591645:0] socket: 689 qp: -1
2025-11-11T17:21:24.316243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519182019432479:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:24.316280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:24.316566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519177724465086:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:24.316791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519177724465084:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519182019432463:2118] held: no
2025-11-11T17:21:24.318155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519177724465085:2049] [node 2] ICP22 created new session: [6:7571519182019432478:2049]
2025-11-11T17:21:24.318211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519182019432478:2049] [node 2] ICS09 handshake done sender: [6:7571519182019432460:2116] self: [6:7571519182019432462:0] peer: [2:7571519178179160140:0] socket: 671 qp: -1
2025-11-11T17:21:24.318251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519182019432478:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:24.318303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:24.327516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519177724465085:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20976049ns) {HandleHandshakeStatus:329(20972910ns) {DropIncomingHandshake:441(620ns) DropOutgoingHandshake:456(650ns) SwitchToState:178(831ns) ProcessPendingSessionEvents:495(1044ns)}}
2025-11-11T17:21:24.056170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519178179160059:2048] [node 1] ICP22 created new session: [2:7571519182474127449:2048]
2025-11-11T17:21:24.056217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519182474127449:2048] [node 1] ICS09 handshake done sender: [2:7571519178179160136:2115] self: [2:7571519178179160139:0] peer: [1:7571519175852343198:0] socket: 653 qp: -1
2025-11-11T17:21:24.124385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519180118390486:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:24.079123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:24.342039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519182019432476:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26465
2025-11-11T17:21:24.056258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519182474127449:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:24.056293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:24.060425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:24.062102Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519178179160130:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:24.080375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519178179160122:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.080412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:24.080462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519178179160122:2108] [node 5] ICP22 created new session: [2:7571519182474127452:2108]
2025-11-11T17:21:24.080507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519182474127452:2108] [node 5] ICS09 handshake done sender: [2:7571519178179160130:2112] self: [2:7571519178179160129:0] peer: [5:7571519180118390480:0] socket: 641 qp: -1
2025-11-11T17:21:24.080533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519182474127452:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:24.080555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:24.124442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:24.091649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519178179160060:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(86929197ns) {HandleHandshakeStatus:329(86923834ns) {DropIncomingHandshake:441(18957ns) DropOutgoingHandshake:456(501ns) SwitchToState:178(493ns) ProcessPendingSessionEvents:495(7977216ns) {ForwardSessionEventToSession:612(7909412ns) ForwardSessionEventToSession:612(6460ns) ForwardSessionEventToSession:612(8499ns) ForwardSessionEventToSession:612(2281ns) ForwardSessionEventToSession:612(5277ns) ForwardSessionEventToSession:612(3143ns) ForwardSessionEventToSession:612(16342ns) ForwardSessionEventToSession:612(7435ns) ForwardSessionEventToSession:612(3145ns) ForwardSessionEventToSession:612(2306ns)}}}
2025-11-11T17:21:24.111345Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519182474127446:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.132422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:24.119082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160144:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.134008Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.119133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519178179160144:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.134099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519175823423166:2109] [node 4] ICP22 created new session: [5:7571519180118390489:2109]
2025-11-11T17:21:24.125718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519178179160122:2108] [node 5] ICP18 handshake (actor [2:7571519182474127446:2120]) is too old
2025-11-11T17:21:24.135296Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519178179160143:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.135333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519175823423100:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519175823423164:2111] held: no
2025-11-11T17:21:24.135333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519178179160143:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.139023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519180118390489:2109] [node 4] ICS09 handshake done sender: [5:7571519175823423165:2112] self: [5:7571519175823423174:0] peer: [4:7571519175612303730:0] socket: 647 qp: -1
2025-11-11T17:21:24.145148Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519178179160144:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:24.139250Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519180118390483:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.145359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519178179160143:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:24.139645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519180118390489:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:24.148440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519178179160127:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.142976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:24.148544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:24.148593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519178179160127:2109] [node 6] ICP22 created new session: [2:7571519182474127456:2109]
2025-11-11T17:21:24.145622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519175823423099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:24.148639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519182474127456:2109] [node 6] ICS09 handshake done sender: [2:7571519178179160144:2118] self: [2:7571519178179160140:0] peer: [6:7571519182019432462:0] socket: 669 qp: -1
2025-11-11T17:21:24.148682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519182474127456:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:24.151064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519180118390482:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:24.148729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:24.159171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519175823423164:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:24.152298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519178179160126:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.171022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519175823423100:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.170834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:24.186460Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519175823423180:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.175724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519178179160126:2107] [node 4] ICP22 created new session: [2:7571519182474127458:2107]
2025-11-11T17:21:24.186488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519175823423180:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.183744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519182474127458:2107] [node 4] ICS09 handshake done sender: [2:7571519178179160143:2117] self: [2:7571519178179160142:0] peer: [4:7571519179907271051:0] socket: 632 qp: -1
2025-11-11T17:21:24.196096Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519175823423180:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:24.183792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519182474127458:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:24.199669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:24.199252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:24.202144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519175823423100:2049] [node 2] ICP22 created new session: [5:7571519180118390494:2049]
2025-11-11T17:21:24.298813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519178179160126:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(115431714ns) {HandleHandshakeStatus:329(115428327ns) {DropIncomingHandshake:441(19255ns) DropOutgoingHandshake:456(438ns) SwitchToState:178(840ns) ProcessPendingSessionEvents:495(25492552ns) {ForwardSessionEventToSession:612(10315499ns) ForwardSessionEventToSession:612(7722ns) ForwardSessionEventToSession:612(3964ns) ForwardSessionEventToSession:612(6434574ns) ForwardSessionEventToSession:612(4283ns) ForwardSessionEventToSession:612(507881ns) ForwardSessionEventToSession:612(4522ns) ForwardSessionEventToSession:612(3462176ns) ForwardSessionEventToSession:612(3901704ns) ForwardSessionEventToSession:612(6286ns)}}}
2025-11-11T17:21:24.225965Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:24.229036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519175823423163:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.229033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519180118390494:2049] [node 2] ICS09 handshake done sender: [5:7571519175823423164:2111] self: [5:7571519180118390480:0] peer: [2:7571519178179160129:0] socket: 645 qp: -1
2025-11-11T17:21:24.232520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:24.232583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519175823423163:2110] [node 6] ICP22 created new session: [5:7571519180118390495:2110]
2025-11-11T17:21:24.232634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519180118390495:2110] [node 6] ICS09 handshake done sender: [5:7571519175823423180:2118] self: [5:7571519175823423176:0] peer: [6:7571519182019432470:0] socket: 662 qp: -1
2025-11-11T17:21:24.232673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519180118390495:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:24.233048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519175823423166:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(64080447ns) {HandleHandshakeStatus:329(64076251ns) {DropIncomingHandshake:441(713ns) DropOutgoingHandshake:456(5567ns) SwitchToState:178(599ns) ProcessPendingSessionEvents:495(23560010ns) {ForwardSessionEventToSession:612(5613298ns) ForwardSessionEventToSession:612(5914456ns) ForwardSessionEventToSession:612(4572ns) ForwardSessionEventToSession:612(3680ns) ForwardSessionEventToSession:612(7004535ns) ForwardSessionEventToSession:612(3083ns) ForwardSessionEventToSession:612(255653ns) ForwardSessionEventToSession:612(2761020ns) ForwardSessionEventToSession:612(4064ns) ForwardSessionEventToSession:612(1971636ns)}}}
2025-11-11T17:21:24.081209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519180147310511:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37433
2025-11-11T17:21:24.236565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519180118390494:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:24.236612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:24.081355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519180147310512:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37433
2025-11-11T17:21:24.236793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:24.085706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519175852343126:2048] [node 2] ICP22 created new session: [1:7571519180147310510:2048]
2025-11-11T17:21:24.085763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519180147310510:2048] [node 2] ICS09 handshake done sender: [1:7571519175852343199:2113] self: [1:7571519175852343198:0] peer: [2:7571519178179160139:0] socket: 652 qp: -1
2025-11-11T17:21:24.237237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519175823423163:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(17850975ns) {HandleHandshakeStatus:329(17846532ns) {DropIncomingHandshake:441(22416ns) DropOutgoingHandshake:456(551ns) SwitchToState:178(830ns) ProcessPendingSessionEvents:495(105459ns) {ForwardSessionEventToSession:612(62207ns) ForwardSessionEventToSession:612(6620ns) ForwardSessionEventToSession:612(2017ns) ForwardSessionEventToSession:612(2598ns) ForwardSessionEventToSession:612(7142ns) ForwardSessionEventToSession:612(2064ns) ForwardSessionEventToSession:612(2758ns) ForwardSessionEventToSession:612(5179ns) ForwardSessionEventToSession:612(2011ns) ForwardSessionEventToSession:612(4779ns)}}}
2025-11-11T17:21:24.085803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519180147310510:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:24.239445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519175823423099:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519180118390483:2121] held: no
2025-11-11T17:21:24.085862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:24.258109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519175823423100:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(79904243ns) {HandleHandshakeStatus:329(79901109ns) {DropIncomingHandshake:441(1181ns) DropOutgoingHandshake:456(295ns) SwitchToState:178(521ns) ProcessPendingSessionEvents:495(1258ns)}}
2025-11-11T17:21:24.086271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519175852343126:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(24344481ns) {HandleHandshakeStatus:329(24341784ns) {DropIncomingHandshake:441(11519ns) DropOutgoingHandshake:456(240ns) SwitchToState:178(223ns) ProcessPendingSessionEvents:495(96286ns) {ForwardSessionEventToSession:612(51673ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(2298ns) ForwardSessionEventToSession:612(4739ns) ForwardSessionEventToSession:612(2400ns) ForwardSessionEventToSession:612(3065ns) ForwardSessionEventToSession:612(5762ns) ForwardSessionEventToSession:612(5128ns) ForwardSessionEventToSession:612(6107ns) ForwardSessionEventToSession:612(3209ns)}}}
2025-11-11T17:21:24.261738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519180118390492:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36628
2025-11-11T17:21:24.097730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519180147310511:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.288009Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519180118390483:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:24.109782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519175852343187:2107] [node 4] ICP18 handshake (actor [1:7571519180147310511:2118]) is too old
2025-11-11T17:21:24.297935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519175823423099:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.317734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:24.321669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519175823423099:2048] [node 1] ICP22 created new session: [5:7571519180118390498:2048]
2025-11-11T17:21:24.329701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519180118390498:2048] [node 1] ICS09 handshake done sender: [5:7571519180118390483:2121] self: [5:7571519180118390491:0] peer: [1:7571519175852343204:0] socket: 638 qp: -1
2025-11-11T17:21:24.329756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519180118390498:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:24.329846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:24.330088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519175823423099:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32075638ns) {HandleHandshakeStatus:329(32073248ns) {DropIncomingHandshake:441(478ns) DropOutgoingHandshake:456(254ns) SwitchToState:178(464ns) ProcessPendingSessionEvents:495(1220ns)}}
2025-11-11T17:21:24.128645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519180147310512:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.317691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519180750558963:2109] [node 6] ICS09 handshake done sender: [3:7571519176455591647:2117] self: [3:7571519176455591645:0] peer: [6:7571519182019432475:0] socket: 673 qp: -1
2025-11-11T17:21:24.153804Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343205:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.153847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519175852343205:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.179742Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519175852343205:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:24.183763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519175852343190:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.219742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:24.229166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519175852343190:2108] [node 5] ICP22 created new session: [1:7571519180147310522:2108]
2025-11-11T17:21:24.236525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519180147310522:2108] [node 5] ICS09 handshake done sender: [1:7571519175852343205:2115] self: [1:7571519175852343204:0] peer: [5:7571519180118390491:0] socket: 685 qp: -1
2025-11-11T17:21:24.240120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519180147310522:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:24.240223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:24.240715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519175852343190:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(56728689ns) {HandleHandshakeStatus:329(56726049ns) {DropIncomingHandshake:441(26143ns) DropOutgoingHandshake:456(689ns) SwitchToState:178(409ns) ProcessPendingSessionEvents:495(119499ns) {ForwardSessionEventToSession:612(57621ns) ForwardSessionEventToSession:612(8701ns) ForwardSessionEventToSession:612(2291ns) ForwardSessionEventToSession:612(4851ns) ForwardSessionEventToSession:612(2571ns) ForwardSessionEventToSession:612(5106ns) ForwardSessionEventToSession:612(2578ns) ForwardSessionEventToSession:612(2356ns) ForwardSessionEventToSession:612(5236ns) ForwardSessionEventToSession:612(18166ns)}}}
2025-11-11T17:21:24.249969Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519175852343208:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:24.249997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519175852343208:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:24.257824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519175852343208:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:24.278173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519175852343191:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:24.297812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:24.298751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519175852343191:2109] [node 6] ICP22 created new session: [1:7571519180147310525:2109]
2025-11-11T17:21:24.299568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519180147310525:2109] [node 6] ICS09 handshake done sender: [1:7571519175852343208:2117] self: [1:7571519175852343207:0] peer: [6:7571519182019432474:0] socket: 684 qp: -1
2025-11-11T17:21:24.301941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519180147310525:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:24.306611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:24.322134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519175852343191:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(47718368ns) {HandleHandshakeStatus:329(47714212ns) {DropIncomingHandshake:441(38428ns) DropOutgoingHandshake:456(435ns) SwitchToState:178(858ns) ProcessPendingSessionEvents:495(127227ns) {ForwardSessionEventToSession:612(69793ns) ForwardSessionEventToSession:612(3449ns) ForwardSessionEventToSession:612(6742ns) ForwardSessionEventToSession:612(4148ns) ForwardSessionEventToSession:612(9708ns) ForwardSessionEventToSession:612(5183ns) ForwardSessionEventToSession:612(3140ns) ForwardSessionEventToSession:612(12638ns) ForwardSessionEventToSession:612(2910ns) ForwardSessionEventToSession:612(2538ns)}}}
2025-11-11T17:21:24.348057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519182019432463:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:24.318804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519180750558963:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:24.364641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519177724465084:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:24.370486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:24.325754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:24.384480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519177724465084:2048] [node 1] ICP22 created new session: [6:7571519182019432482:2048]
2025-11-11T17:21:24.354179Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519176455591631:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(75517477ns) {HandleHandshakeStatus:329(75514260ns) {DropIncomingHandshake:441(28361ns) DropOutgoingHandshake:456(586ns) SwitchToState:178(1013ns) ProcessPendingSessionEvents:495(8126868ns) {ForwardSessionEventToSession:612(106081ns) ForwardSessionEventToSession:612(3602ns) ForwardSessionEventToSession:612(3034ns) ForwardSessionEventToSession:612(6046ns) ForwardSessionEventToSession:612(4096ns) ForwardSessionEventToSession:612(3375ns) ForwardSessionEventToSession:612(7963890ns) ForwardSessionEventToSession:612(3644ns) ForwardSessionEventToSession:612(15504ns) ForwardSessionEventToSession:612(2823ns)}}}
2025-11-11T17:21:24.384565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519182019432482:2048] [node 1] ICS09 handshake done sender: [6:7571519182019432463:2118] self: [6:7571519182019432474:0] peer: [1:7571519175852343207:0] socket: 680 qp: -1
2025-11-11T17:21:24.384613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519182019432482:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:24.384673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:24.384999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519177724465084:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20287195ns) {HandleHandshakeStatus:329(20283282ns) {DropIncomingHandshake:441(1422ns) DropOutgoingHandshake:456(509ns) SwitchToState:178(866ns) ProcessPendingSessionEvents:495(1132ns)}}
2025-11-11T17:21:24.427534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:24.427720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519182474127449:2048] [node 1] ICS01 socket: 653 reason# EndOfStream
2025-11-11T17:21:24.427747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519178179160059:2048] [node 1] ICP30 unregister session Session# [2:7571519182474127449:2048] VirtualId# [2:7571519178179160139:0]
2025-11-11T17:21:24.427777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519182474127449:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.427897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:24.427947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:24.430004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:24.430242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519180750558948:2048] [node 1] ICS07 socket disconnect 650 reason# EndOfStream
2025-11-11T17:21:24.430272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519180750558948:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.430373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:24.430427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519180750558948:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:24.433849Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:21:24.434798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519176455591564:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:24.434854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519180750558948:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:24.435639Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Couldn't connect to any resolved address
2025-11-11T17:21:24.435867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519176455591564:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:24.435921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519180750558948:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:24.436949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519176455591564:2048] [node 1] ICP30 unregister session Session# [3:7571519180750558948:2048] VirtualId# [3:7571519176455591644:0]
2025-11-11T17:21:24.437052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:24.437099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519176455591564:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:24.437122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:24.440950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:24.440638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:24.441140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519182019432482:2048] [node 1] ICS01 socket: 680 reason# EndOfStream
2025-11-11T17:21:24.441175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519177724465084:2048] [node 1] ICP30 unregister session Session# [6:7571519182019432482:2048] VirtualId# [6:7571519182019432474:0]
2025-11-11T17:21:24.441703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:24.441202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519182019432482:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.441281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:24.441317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:24.440868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519179907271047:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-11-11T17:21:24.440907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519175612303661:2048] [node 1] ICP30 unregister session Session# [4:7571519179907271047:2048] VirtualId# [4:7571519175612303738:0]
2025-11-11T17:21:24.440945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519179907271047:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.441070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:24.441123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:24.442393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519180118390498:2048] [node 1] ICS01 socket: 638 reason# EndOfStream
2025-11-11T17:21:24.442440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519175823423099:2048] [node 1] ICP30 unregister session Session# [5:7571519180118390498:2048] VirtualId# [5:7571519180118390491:0]
2025-11-11T17:21:24.442476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519180118390498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.442586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:24.442644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:24.447459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519176455591564:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:24.519775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:24.520061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519180750558951:2049] [node 2] ICS01 socket: 655 reason# EndOfStream
2025-11-11T17:21:24.520142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519176455591565:2049] [node 2] ICP30 unregister session Session# [3:7571519180750558951:2049] VirtualId# [3:7571519176455591641:0]
2025-11-11T17:21:24.520186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519180750558951:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.520273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:24.520325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:24.522867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:24.523087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519179907271055:2049] [node 2] ICS01 socket: 683 reason# EndOfStream
2025-11-11T17:21:24.523139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519175612303662:2049] [node 2] ICP30 unregister session Session# [4:7571519179907271055:2049] VirtualId# [4:7571519179907271051:0]
2025-11-11T17:21:24.523170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519179907271055:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.523243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:24.523283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:24.524209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:24.524531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519180118390494:2049] [node 2] ICS01 socket: 645 reason# EndOfStream
2025-11-11T17:21:24.524578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519175823423100:2049] [node 2] ICP30 unregister session Session# [5:7571519180118390494:2049] VirtualId# [5:7571519180118390480:0]
2025-11-11T17:21:24.524608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519180118390494:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.524678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:24.524726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:24.525435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:24.525624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519182019432478:2049] [node 2] ICS01 socket: 671 reason# EndOfStream
2025-11-11T17:21:24.525665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519177724465085:2049] [node 2] ICP30 unregister session Session# [6:7571519182019432478:2049] VirtualId# [6:7571519182019432462:0]
2025-11-11T17:21:24.525697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519182019432478:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.525751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:24.525781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:24.606574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:24.606837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519179907271057:2050] [node 3] ICS01 socket: 672 reason# EndOfStream
2025-11-11T17:21:24.606888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519175612303663:2050] [node 3] ICP30 unregister session Session# [4:7571519179907271057:2050] VirtualId# [4:7571519179907271049:0]
2025-11-11T17:21:24.606934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519179907271057:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.607015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:24.607063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:24.611060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:24.611285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519180118390486:2050] [node 3] ICS01 socket: 654 reason# EndOfStream
2025-11-11T17:21:24.611328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519175823423101:2050] [node 3] ICP30 unregister session Session# [5:7571519180118390486:2050] VirtualId# [5:7571519180118390479:0]
2025-11-11T17:21:24.611365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519180118390486:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.611462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:24.611501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:24.614936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:24.615155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519182019432479:2050] [node 3] ICS01 socket: 689 reason# EndOfStream
2025-11-11T17:21:24.615192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519177724465086:2050] [node 3] ICP30 unregister session Session# [6:7571519182019432479:2050] VirtualId# [6:7571519182019432475:0]
2025-11-11T17:21:24.615226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519182019432479:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.615324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:24.615370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:24.699737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:24.699999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519180118390489:2109] [node 4] ICS07 socket disconnect 647 reason# EndOfStream
2025-11-11T17:21:24.700039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519180118390489:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.700136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:24.700182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519180118390489:2109] [node 4] ICS15 start handshake
2025-11-11T17:21:24.701132Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address
2025-11-11T17:21:24.701399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519175823423166:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:24.701450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519180118390489:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:24.701489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519175823423166:2109] [node 4] ICP30 unregister session Session# [5:7571519180118390489:2109] VirtualId# [5:7571519175823423174:0]
2025-11-11T17:21:24.701532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:24.701565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519175823423166:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:24.701579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:24.702639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:24.702954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519182019432465:2107] [node 4] ICS07 socket disconnect 636 reason# EndOfStream
2025-11-11T17:21:24.703017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519182019432465:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.703170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:24.703278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519182019432465:2107] [node 4] ICS15 start handshake
2025-11-11T17:21:24.716409Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address
2025-11-11T17:21:24.716730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519177724465149:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:24.716784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519182019432465:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:24.716826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519177724465149:2107] [node 4] ICP30 unregister session Session# [6:7571519182019432465:2107] VirtualId# [6:7571519182019432459:0]
2025-11-11T17:21:24.716881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:24.716908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519177724465149:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:24.716930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:24.727950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519177724465149:2107] [node 4] ICP33 wake up from error state
2025-11-11T17:21:24.769785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:24.770041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519182019432472:2099] [node 5] ICS01 socket: 674 reason# EndOfStream
2025-11-11T17:21:24.770084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519177724465142:2099] [node 5] ICP30 unregister session Session# [6:7571519182019432472:2099] VirtualId# [6:7571519182019432470:0]
2025-11-11T17:21:24.770127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519182019432472:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:24.770245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:24.770295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-11T17:21:24.940324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519181354245847:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:24.940371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519181354245848:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:24.963066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519182168761456:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:24.963114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519182168761457:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:24.985217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519180481743024:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:24.985264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519180481743025:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:25.004189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519187999839076:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:25.004248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519187999839077:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:25.004299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519187999839078:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:25.029271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519185390428530:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:25.029328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519185390428531:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:25.029375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519185390428532:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:25.054349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519186024993178:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:25.054397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519186024993179:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:25.054421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519186024993180:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:25.245715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519186024993237:2106] [node 5] ICP01 ready to work
2025-11-11T17:21:25.322686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519184776710384:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:25.323172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519186463728815:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:25.323462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519184776710385:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:25.323626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519186463728816:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:25.323317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519185390428594:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:25.324042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519185390428595:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:25.324158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519184776710386:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:25.324513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519186463728817:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:25.324750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519181354245847:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(90723449ns) {Configure:104(90521686ns) {ProcessConfigured:135(83993ns) {StartInitialHandshake:157(80371ns) {DropHandshakes:469(3174ns) {DropIncomingHandshake:441(741ns) DropOutgoingHandshake:456(623ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:476(2352ns) {SwitchToState:178(1003ns)}}}}}
2025-11-11T17:21:25.325426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519185649213208:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:25.325552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519186024993244:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:25.323178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519187999839139:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:25.323885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519187999839140:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:25.330738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519185649213209:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:25.346116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519185649213210:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:25.347612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519182168761457:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16685841ns) {Configure:104(16678103ns) {ProcessConfigured:135(88218ns) {StartInitialHandshake:157(85308ns) {DropHandshakes:469(5011ns) {DropIncomingHandshake:441(511ns) DropOutgoingHandshake:456(338ns)} GenerateSessionVirtualId:988(1003ns) PrepareNewSessionHandshake:476(14224ns) {SwitchToState:178(447ns)}}}}}
2025-11-11T17:21:25.351543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519180481743024:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26330642ns) {Configure:104(26325302ns) {ProcessConfigured:135(329728ns) {StartInitialHandshake:157(256296ns) {DropHandshakes:469(1794ns) {DropIncomingHandshake:441(337ns) DropOutgoingHandshake:456(522ns)} GenerateSessionVirtualId:988(6184ns) PrepareNewSessionHandshake:476(1236ns) {SwitchToState:178(508ns)}}}}}
2025-11-11T17:21:25.350835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519187999839076:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17472610ns) {Configure:104(17468093ns) {ProcessConfigured:135(11062570ns) {StartInitialHandshake:157(11059703ns) {DropHandshakes:469(1864ns) {DropIncomingHandshake:441(542ns) DropOutgoingHandshake:456(628ns)} GenerateSessionVirtualId:988(606ns) PrepareNewSessionHandshake:476(6104ns) {SwitchToState:178(3250ns)}}}}}
2025-11-11T17:21:25.352111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519185390428530:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26475708ns) {Configure:104(26470792ns) {ProcessConfigured:135(342856ns) {StartInitialHandshake:157(340872ns) {DropHandshakes:469(699ns) {DropIncomingHandshake:441(162ns) DropOutgoingHandshake:456(267ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:476(801ns) {SwitchToState:178(331ns)}}}}}
2025-11-11T17:21:25.352318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519186024993178:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21195880ns) {Configure:104(21189085ns) {ProcessConfigured:135(208925ns) {StartInitialHandshake:157(205643ns) {DropHandshakes:469(93095ns) {DropIncomingHandshake:441(632ns) DropOutgoingHandshake:456(237ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:476(1235ns) {SwitchToState:178(581ns)}}}}}
2025-11-11T17:21:25.380182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519187999839077:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21332951ns) {Configure:104(21327437ns) {ProcessConfigured:135(12309221ns) {StartInitialHandshake:157(12305404ns) {DropHandshakes:469(1200ns) {DropIncomingHandshake:441(261ns) DropOutgoingHandshake:456(251ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:476(5758ns) {SwitchToState:178(3437ns)}}}}}
2025-11-11T17:21:25.392728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519186024993179:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(39176881ns) {Configure:104(39172768ns) {ProcessConfigured:135(11170357ns) {StartInitialHandshake:157(11166180ns) {DropHandshakes:469(2636ns) {DropIncomingHandshake:441(601ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(867ns) PrepareNewSessionHandshake:476(4925ns) {SwitchToState:178(2662ns)}}}}}
2025-11-11T17:21:25.393319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519184776710385:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(20705155ns) {Configure:104(20700393ns) {ProcessConfigured:135(44281ns) {StartInitialHandshake:157(42481ns) {DropHandshakes:469(2524ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(469ns)} GenerateSessionVirtualId:988(1209ns) PrepareNewSessionHandshake:476(799ns) {SwitchToState:178(242ns)}}}}}
2025-11-11T17:21:25.394932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519185649213210:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32804454ns) {Configure:104(32800106ns) {ProcessConfigured:135(865332ns) {StartInitialHandshake:157(356826ns) {DropHandshakes:469(2252ns) {DropIncomingHandshake:441(346ns) DropOutgoingHandshake:456(335ns)} GenerateSessionVirtualId:988(1131ns) PrepareNewSessionHandshake:476(2199ns) {SwitchToState:178(1232ns)}}}}}
2025-11-11T17:21:25.394876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519186463728817:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(39821869ns) {Configure:104(39818422ns) {ProcessConfigured:135(4013529ns) {StartInitialHandshake:157(4010536ns) {DropHandshakes:469(622984ns) {DropIncomingHandshake:441(589ns) DropOutgoingHandshake:456(1627ns)} GenerateSessionVirtualId:988(2006ns) PrepareNewSessionHandshake:476(2381ns) {SwitchToState:178(1202ns)}}}}}
2025-11-11T17:21:25.416286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519184776710384:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19437625ns) {Configure:104(19435070ns) {ProcessConfigured:135(53137ns) {StartInitialHandshake:157(50093ns) {DropHandshakes:469(1829ns) {DropIncomingHandshake:441(352ns) DropOutgoingHandshake:456(209ns)} GenerateSessionVirtualId:988(565ns) PrepareNewSessionHandshake:476(892ns) {SwitchToState:178(267ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.428287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519184776710395:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20831
2025-11-11T17:21:25.428455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519187999839139:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(26681071ns) {Configure:104(26674805ns) {ProcessConfigured:135(229247ns) {StartInitialHandshake:157(225829ns) {DropHandshakes:469(19458ns) {DropIncomingHandshake:441(656ns) DropOutgoingHandshake:456(514ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:476(2462ns) {SwitchToState:178(660ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.430157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519185649213215:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64871
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.432028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519186463728828:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55812
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.432734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519185649213221:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64871
2025-11-11T17:21:25.432797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519184776710401:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20831
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.433258Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710395:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.433902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519186463728832:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55812
2025-11-11T17:21:25.433970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519186463728833:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55812
2025-11-11T17:21:25.433640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519185649213224:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64871
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.447153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993253:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.452687Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213215:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.453096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519185649213225:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64871
2025-11-11T17:21:25.453243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519185649213226:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64871
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.470498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428609:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.478704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519184776710402:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20831
2025-11-11T17:21:25.488705Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728828:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.489335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519186463728834:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55812
2025-11-11T17:21:25.471989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428610:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.493149Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710392:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.493205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519184776710392:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.472072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428611:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.494107Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213221:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.489435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519186463728835:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55812
2025-11-11T17:21:25.495187Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710401:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.472501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428612:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.495328Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728832:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.486446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519185390428609:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.497651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519180481743025:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.500134Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728821:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.499569Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213224:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.486657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519185390428532:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.501215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519186024993253:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.500169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519186463728821:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.502707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213226:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.503034Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213214:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.503060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519185649213214:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.499360Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519184776710392:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:25.501468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519186024993178:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.505205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993258:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.505076Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519185649213214:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:25.489985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519185390428610:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.493287Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519185390428611:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.509611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519184776710385:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.509749Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728833:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.514378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519185649213210:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.513722Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728823:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.509711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:25.509781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519184776710385:2108] [node 5] ICP22 created new session: [3:7571519184776710406:2108]
2025-11-11T17:21:25.509844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519184776710406:2108] [node 5] ICS09 handshake done sender: [3:7571519184776710392:2113] self: [3:7571519184776710391:0] peer: [5:7571519185390428613:0] socket: 45 qp: -1
2025-11-11T17:21:25.509891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519184776710406:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:25.509943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:25.513778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519186463728823:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.516853Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728834:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.517267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519182168761456:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.517628Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519186463728821:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:25.498724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519185390428531:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.499974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519185390428530:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.514473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:25.514508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519185649213210:2109] [node 6] ICP22 created new session: [1:7571519185649213232:2109]
2025-11-11T17:21:25.519597Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519185390428612:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.519730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710402:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.519890Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24401) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.518180Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519186024993253:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:25.520644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519186024993178:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.520759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:25.520814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519186024993178:2048] [node 1] ICP22 created new session: [6:7571519186024993259:2048]
2025-11-11T17:21:25.520858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519186024993259:2048] [node 1] ICS09 handshake done sender: [6:7571519186024993253:2115] self: [6:7571519186024993257:0] peer: [1:7571519185649213213:0] socket: 64 qp: -1
2025-11-11T17:21:25.520896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519186024993259:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:25.520932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:25.522777Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.514604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519185649213232:2109] [node 6] ICS09 handshake done sender: [1:7571519185649213214:2112] self: [1:7571519185649213213:0] peer: [6:7571519186024993257:0] socket: 57 qp: -1
2025-11-11T17:21:25.516196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519185649213232:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:25.523209Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519186463728823:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:25.516230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:25.554758Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728835:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.557760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519182168761457:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.557873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:25.557945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519182168761457:2049] [node 3] ICP22 created new session: [2:7571519186463728842:2049]
2025-11-11T17:21:25.558008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519186463728842:2049] [node 3] ICS09 handshake done sender: [2:7571519186463728821:2111] self: [2:7571519186463728820:0] peer: [3:7571519184776710405:0] socket: 48 qp: -1
2025-11-11T17:21:25.561009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519186463728842:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:25.561085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:25.518692Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213225:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.562494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519186024993178:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.563381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519184776710408:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20831
2025-11-11T17:21:25.520451Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213207:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.566825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519186463728815:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.566900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:25.566933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519186463728815:2107] [node 5] ICP22 created new session: [2:7571519186463728844:2107]
2025-11-11T17:21:25.566966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519186463728844:2107] [node 5] ICS09 handshake done sender: [2:7571519186463728823:2112] self: [2:7571519186463728822:0] peer: [5:7571519185390428614:0] socket: 52 qp: -1
2025-11-11T17:21:25.566993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519186463728844:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:25.567018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:25.520496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519185649213207:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.575173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428615:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.525773Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213212:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.525797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519185649213212:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.537774Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519185649213207:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:25.538316Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519185649213212:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:25.538565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519181354245847:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.575927Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.538683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-11-11T17:21:25.538737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519181354245847:2048] [node 2] ICP22 created new session: [1:7571519185649213238:2048]
::ffff:127.1.0.0
2025-11-11T17:21:25.538791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519185649213238:2048] [node 2] ICS09 handshake done sender: [1:7571519185649213207:2110] self: [1:7571519185649213206:0] peer: [2:7571519186463728840:0] socket: 47 qp: -1
2025-11-11T17:21:25.538836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519185649213238:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:25.576341Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.538871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:25.576121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993261:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.574086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839154:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.585509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519185390428532:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24401) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519185390428609:2116] held: no
2025-11-11T17:21:25.575431Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519187999839154:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.559373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519181354245847:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(19949555ns) {HandleHandshakeStatus:329(19947053ns) {DropIncomingHandshake:441(16810ns) DropOutgoingHandshake:456(738ns) SwitchToState:178(1214ns) ProcessPendingSessionEvents:495(19492947ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19420842ns max# 7986677ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.591669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519184776710410:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20831
2025-11-11T17:21:25.559501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519185649213209:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.575566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519187999839076:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.588475Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.591811Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519184776710401:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.590921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839156:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.592784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519180481743025:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519184776710401:2117] held: no
2025-11-11T17:21:25.593334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519180481743025:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.593381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:25.593435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519180481743025:2049] [node 2] ICP22 created new session: [3:7571519184776710413:2049]
2025-11-11T17:21:25.593483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519184776710413:2049] [node 2] ICS09 handshake done sender: [3:7571519184776710401:2117] self: [3:7571519184776710405:0] peer: [2:7571519186463728820:0] socket: 50 qp: -1
2025-11-11T17:21:25.593517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519184776710413:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:25.593546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:25.597065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519184776710412:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20831
2025-11-11T17:21:25.602886Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519186024993261:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.559641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:25.559694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519185649213209:2108] [node 5] ICP22 created new session: [1:7571519185649213240:2108]
2025-11-11T17:21:25.603366Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519187999839154:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:25.559738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519185649213240:2108] [node 5] ICS09 handshake done sender: [1:7571519185649213212:2111] self: [1:7571519185649213211:0] peer: [5:7571519185390428616:0] socket: 54 qp: -1
2025-11-11T17:21:25.606698Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710412:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.603836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519187999839076:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.629712Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.621351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519186024993180:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.559779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519185649213240:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:25.616132Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710410:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.603895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:25.645957Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519185390428609:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-11T17:21:25.616330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519180481743024:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.632522Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710394:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.632554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519184776710394:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.643487Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519184776710394:2114] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-11T17:21:25.603948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519187999839076:2048] [node 1] ICP22 created new session: [4:7571519187999839158:2048]
2025-11-11T17:21:25.559838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:25.648236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519182168761456:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519186463728833:2117] held: no
2025-11-11T17:21:25.650694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993264:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.650407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519184776710386:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.650761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:25.603991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519187999839158:2048] [node 1] ICS09 handshake done sender: [4:7571519187999839154:2115] self: [4:7571519187999839155:0] peer: [1:7571519185649213218:0] socket: 49 qp: -1
2025-11-11T17:21:25.577300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213219:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.653102Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519186024993261:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:25.651141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519184776710386:2109] [node 6] ICP22 created new session: [3:7571519184776710419:2109]
2025-11-11T17:21:25.651607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519184776710419:2109] [node 6] ICS09 handshake done sender: [3:7571519184776710394:2114] self: [3:7571519184776710393:0] peer: [6:7571519186024993263:0] socket: 81 qp: -1
2025-11-11T17:21:25.604024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519187999839158:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.654914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519186463728846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55812
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.655806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519185390428530:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519185390428611:2118] held: no
2025-11-11T17:21:25.577328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519185649213219:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.651644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519184776710419:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:25.659412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428617:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.659756Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24401) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.651695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:25.604106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:25.605056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839157:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.659300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519187999839157:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.585435Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519185649213219:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:25.659026Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519186463728833:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:25.591380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519185649213208:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.661001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519182168761456:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.661052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:25.661119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519182168761456:2048] [node 1] ICP22 created new session: [2:7571519186463728850:2048]
2025-11-11T17:21:25.661169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519186463728850:2048] [node 1] ICS09 handshake done sender: [2:7571519186463728833:2117] self: [2:7571519186463728840:0] peer: [1:7571519185649213206:0] socket: 51 qp: -1
2025-11-11T17:21:25.661208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519186463728850:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:25.602670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:25.602771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519185649213208:2107] [node 4] ICP22 created new session: [1:7571519185649213244:2107]
2025-11-11T17:21:25.602822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519185649213244:2107] [node 4] ICS09 handshake done sender: [1:7571519185649213219:2115] self: [1:7571519185649213218:0] peer: [4:7571519187999839155:0] socket: 39 qp: -1
2025-11-11T17:21:25.602884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519185649213244:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:25.605618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:25.651423Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519185649213217:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.651452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519185649213217:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.664918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519187999839078:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.665025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839161:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.665440Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519185649213217:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.666630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519186024993180:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.666720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:25.666768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519186024993180:2050] [node 3] ICP22 created new session: [6:7571519186024993266:2050]
2025-11-11T17:21:25.666812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519186024993266:2050] [node 3] ICS09 handshake done sender: [6:7571519186024993261:2117] self: [6:7571519186024993263:0] peer: [3:7571519184776710393:0] socket: 82 qp: -1
2025-11-11T17:21:25.666872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519186024993266:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:25.666905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:25.667719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519186024993180:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:25.669187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993265:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.670064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519181354245848:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.670184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:25.670235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519181354245848:2049] [node 3] ICP22 created new session: [1:7571519185649213247:2049]
2025-11-11T17:21:25.670280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519185649213247:2049] [node 3] ICS09 handshake done sender: [1:7571519185649213217:2114] self: [1:7571519185649213216:0] peer: [3:7571519184776710417:0] socket: 79 qp: -1
2025-11-11T17:21:25.670318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519185649213247:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:25.670382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.671985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519184776710397:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.672020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519184776710397:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.674153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519184776710421:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20831
2025-11-11T17:21:25.674845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519185390428531:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519185390428610:2117] held: no
2025-11-11T17:21:25.675508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519185390428532:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.675585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:25.675633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519185390428532:2050] [node 3] ICP22 created new session: [5:7571519185390428620:2050]
2025-11-11T17:21:25.675688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519185390428620:2050] [node 3] ICS09 handshake done sender: [5:7571519185390428609:2116] self: [5:7571519185390428613:0] peer: [3:7571519184776710391:0] socket: 55 qp: -1
2025-11-11T17:21:25.675730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519185390428620:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:25.675765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:25.677422Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519185390428610:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.680774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839163:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.680839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839164:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.677836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:25.678080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519182168761456:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16997831ns) {HandleHandshakeStatus:329(16995742ns) {DropIncomingHandshake:441(986ns) DropOutgoingHandshake:456(337ns) SwitchToState:178(798ns) ProcessPendingSessionEvents:495(308ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.684223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428618:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.687679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519187999839163:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.689094Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519187999839161:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.689218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519187999839077:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.690075Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.697374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839167:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.690219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519185390428531:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.690266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:25.694603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519185390428531:2049] [node 2] ICP22 created new session: [5:7571519185390428625:2049]
2025-11-11T17:21:25.694675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519185390428625:2049] [node 2] ICS09 handshake done sender: [5:7571519185390428610:2117] self: [5:7571519185390428614:0] peer: [2:7571519186463728822:0] socket: 56 qp: -1
2025-11-11T17:21:25.694719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519185390428625:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:25.694766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:25.696182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428622:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-11-11T17:21:25.696411Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519185390428611:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:25.698511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519180481743024:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519184776710410:2120] held: no
2025-11-11T17:21:25.699246Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519184776710397:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.699760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993269:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.700399Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519187999839157:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:25.700733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519187999839078:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.700786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:25.700835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519187999839078:2050] [node 3] ICP22 created new session: [4:7571519187999839168:2050]
2025-11-11T17:21:25.700882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519187999839168:2050] [node 3] ICS09 handshake done sender: [4:7571519187999839157:2117] self: [4:7571519187999839162:0] peer: [3:7571519184776710396:0] socket: 84 qp: -1
2025-11-11T17:21:25.700917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519187999839168:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:25.700951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:25.702057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519184776710410:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:25.702727Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728825:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.711065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519187999839170:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-11-11T17:21:25.711764Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519187999839161:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.712556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519187999839077:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.712615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:25.712661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519187999839077:2049] [node 2] ICP22 created new session: [4:7571519187999839172:2049]
2025-11-11T17:21:25.712712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519187999839172:2049] [node 2] ICS09 handshake done sender: [4:7571519187999839161:2118] self: [4:7571519187999839166:0] peer: [2:7571519186463728824:0] socket: 86 qp: -1
2025-11-11T17:21:25.712753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519187999839172:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:25.712785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:25.705260Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519186024993265:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.706104Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519185390428622:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.702765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519186463728825:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.706155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519185390428530:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.706201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:25.706253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519185390428530:2048] [node 1] ICP22 created new session: [5:7571519185390428627:2048]
2025-11-11T17:21:25.706295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519185390428627:2048] [node 1] ICS09 handshake done sender: [5:7571519185390428611:2118] self: [5:7571519185390428616:0] peer: [1:7571519185649213211:0] socket: 58 qp: -1
2025-11-11T17:21:25.706328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519185390428627:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:25.706362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:25.717047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519185390428594:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.719303Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519185390428605:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.719330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519185390428605:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.721885Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519185390428605:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:25.722285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519185390428595:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.722346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:25.705394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519186024993179:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.710305Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519186463728825:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:25.712166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993271:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.722398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519185390428595:2109] [node 6] ICP22 created new session: [5:7571519185390428631:2109]
2025-11-11T17:21:25.722453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519185390428631:2109] [node 6] ICS09 handshake done sender: [5:7571519185390428605:2115] self: [5:7571519185390428604:0] peer: [6:7571519186024993272:0] socket: 95 qp: -1
2025-11-11T17:21:25.722492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519185390428631:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:25.722533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:25.731266Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519186463728827:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.732632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519184776710384:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.732702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:25.734481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519184776710384:2107] [node 4] ICP22 created new session: [3:7571519184776710423:2107]
2025-11-11T17:21:25.736273Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519187999839148:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.736345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519187999839148:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.741000Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519187999839148:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:25.731317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519186463728827:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:25.733336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519186463728816:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.733385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:25.733435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519186463728816:2108] [node 4] ICP22 created new session: [2:7571519186463728853:2108]
2025-11-11T17:21:25.733487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519186463728853:2108] [node 4] ICS09 handshake done sender: [2:7571519186463728825:2113] self: [2:7571519186463728824:0] peer: [4:7571519187999839166:0] socket: 43 qp: -1
2025-11-11T17:21:25.733528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519186463728853:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:25.733559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:25.740854Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519186463728827:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:25.718908Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519186024993269:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.719049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519186024993237:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.732161Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.740652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519184776710423:2107] [node 4] ICS09 handshake done sender: [3:7571519184776710397:2116] self: [3:7571519184776710396:0] peer: [4:7571519187999839162:0] socket: 36 qp: -1
2025-11-11T17:21:25.741925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519184776710423:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:25.741977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:25.742653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519180481743024:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.742692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:25.742736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519180481743024:2048] [node 1] ICP22 created new session: [3:7571519184776710425:2048]
2025-11-11T17:21:25.742779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519184776710425:2048] [node 1] ICS09 handshake done sender: [3:7571519184776710410:2120] self: [3:7571519184776710417:0] peer: [1:7571519185649213216:0] socket: 66 qp: -1
2025-11-11T17:21:25.742814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519184776710425:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:25.742844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.744491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519185390428633:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.746731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519186463728817:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.746805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:25.746845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519186463728817:2109] [node 6] ICP22 created new session: [2:7571519186463728856:2109]
2025-11-11T17:21:25.746891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519186463728856:2109] [node 6] ICS09 handshake done sender: [2:7571519186463728827:2114] self: [2:7571519186463728826:0] peer: [6:7571519186024993270:0] socket: 59 qp: -1
2025-11-11T17:21:25.746924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519186463728856:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:25.746954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:25.745218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519186024993237:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519186024993269:2120] held: no
2025-11-11T17:21:25.746320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993273:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.746787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993274:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.748382Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519186024993269:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:25.752181Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519185390428622:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:25.751986Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519187999839167:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.752793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519185390428594:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.752604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519186024993237:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.752946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:25.752996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519186024993237:2106] [node 5] ICP22 created new session: [6:7571519186024993275:2106]
2025-11-11T17:21:25.753225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:25.753273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519185390428594:2108] [node 4] ICP22 created new session: [5:7571519185390428634:2108]
2025-11-11T17:21:25.753315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519185390428634:2108] [node 4] ICS09 handshake done sender: [5:7571519185390428622:2123] self: [5:7571519185390428629:0] peer: [4:7571519187999839147:0] socket: 91 qp: -1
2025-11-11T17:21:25.753353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519185390428634:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:25.753389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:25.753360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519186024993275:2106] [node 5] ICS09 handshake done sender: [6:7571519186024993269:2120] self: [6:7571519186024993272:0] peer: [5:7571519185390428604:0] socket: 42 qp: -1
2025-11-11T17:21:25.753393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519186024993275:2106] [node 5] ICS10 traffic start
2025-11-11T17:21:25.753430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:25.754179Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:25.753981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519187999839139:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.754025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:25.754069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519187999839139:2108] [node 5] ICP22 created new session: [4:7571519187999839175:2108]
2025-11-11T17:21:25.754114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519187999839175:2108] [node 5] ICS09 handshake done sender: [4:7571519187999839148:2113] self: [4:7571519187999839147:0] peer: [5:7571519185390428629:0] socket: 37 qp: -1
2025-11-11T17:21:25.754142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519187999839175:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:25.754166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:25.757081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519187999839139:2108] [node 5] ICP18 handshake (actor [4:7571519187999839167:2121]) is too old
2025-11-11T17:21:25.759942Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519186024993265:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:25.761265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519186024993179:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519186024993265:2119] held: no
2025-11-11T17:21:25.761710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519186024993179:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.762601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:25.762635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519186024993179:2049] [node 2] ICP22 created new session: [6:7571519186024993277:2049]
2025-11-11T17:21:25.762671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519186024993277:2049] [node 2] ICS09 handshake done sender: [6:7571519186024993265:2119] self: [6:7571519186024993270:0] peer: [2:7571519186463728826:0] socket: 53 qp: -1
2025-11-11T17:21:25.762706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519186024993277:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:25.762732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:25.763482Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519186024993271:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.764475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519186024993244:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:25.764814Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519187999839150:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:25.764850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519187999839150:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:25.765874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519186024993280:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36466
2025-11-11T17:21:25.766200Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519187999839150:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:25.766510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519187999839140:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:25.766581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:25.766638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519187999839140:2109] [node 6] ICP22 created new session: [4:7571519187999839178:2109]
2025-11-11T17:21:25.766677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519187999839178:2109] [node 6] ICS09 handshake done sender: [4:7571519187999839150:2114] self: [4:7571519187999839149:0] peer: [6:7571519186024993279:0] socket: 97 qp: -1
2025-11-11T17:21:25.766714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519187999839178:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:25.766745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:25.768089Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519186024993271:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:25.768353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519186024993244:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:25.768426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:25.768468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519186024993244:2109] [node 4] ICP22 created new session: [6:7571519186024993281:2109]
2025-11-11T17:21:25.768499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519186024993281:2109] [node 4] ICS09 handshake done sender: [6:7571519186024993271:2121] self: [6:7571519186024993279:0] peer: [4:7571519187999839149:0] socket: 98 qp: -1
2025-11-11T17:21:25.768523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519186024993281:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:25.768545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:25.798581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:25.798794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519186463728850:2048] [node 1] ICS01 socket: 51 reason# EndOfStream
2025-11-11T17:21:25.798847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519182168761456:2048] [node 1] ICP30 unregister session Session# [2:7571519186463728850:2048] VirtualId# [2:7571519186463728840:0]
2025-11-11T17:21:25.798890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519186463728850:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.798976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:25.799025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:25.799952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:25.800097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519184776710425:2048] [node 1] ICS01 socket: 66 reason# EndOfStream
2025-11-11T17:21:25.800121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519180481743024:2048] [node 1] ICP30 unregister session Session# [3:7571519184776710425:2048] VirtualId# [3:7571519184776710417:0]
2025-11-11T17:21:25.800142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519184776710425:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.800185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:25.800208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:25.805156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:25.805338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519185390428627:2048] [node 1] ICS01 socket: 58 reason# EndOfStream
2025-11-11T17:21:25.805380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519185390428530:2048] [node 1] ICP30 unregister session Session# [5:7571519185390428627:2048] VirtualId# [5:7571519185390428616:0]
2025-11-11T17:21:25.805406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519185390428627:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.805470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:25.805501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:25.805941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:25.806127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519187999839158:2048] [node 1] ICS01 socket: 49 reason# EndOfStream
2025-11-11T17:21:25.806162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519187999839076:2048] [node 1] ICP30 unregister session Session# [4:7571519187999839158:2048] VirtualId# [4:7571519187999839155:0]
2025-11-11T17:21:25.806188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519187999839158:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.806260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:25.806308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:25.810173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:25.810512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519186024993259:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-11-11T17:21:25.810576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519186024993178:2048] [node 1] ICP30 unregister session Session# [6:7571519186024993259:2048] VirtualId# [6:7571519186024993257:0]
2025-11-11T17:21:25.810619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519186024993259:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.810747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:25.810808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:25.861816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:25.862051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519184776710413:2049] [node 2] ICS01 socket: 50 reason# EndOfStream
2025-11-11T17:21:25.862100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519180481743025:2049] [node 2] ICP30 unregister session Session# [3:7571519184776710413:2049] VirtualId# [3:7571519184776710405:0]
2025-11-11T17:21:25.862138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519184776710413:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.862240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:25.862286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:25.864773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:25.865014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519185390428625:2049] [node 2] ICS01 socket: 56 reason# EndOfStream
2025-11-11T17:21:25.865061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519185390428531:2049] [node 2] ICP30 unregister session Session# [5:7571519185390428625:2049] VirtualId# [5:7571519185390428614:0]
2025-11-11T17:21:25.865106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519185390428625:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.865168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:25.865207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:25.867231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:25.867501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519186024993277:2049] [node 2] ICS01 socket: 53 reason# EndOfStream
2025-11-11T17:21:25.867549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519186024993179:2049] [node 2] ICP30 unregister session Session# [6:7571519186024993277:2049] VirtualId# [6:7571519186024993270:0]
2025-11-11T17:21:25.867580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519186024993277:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.867640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:25.867672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:25.869839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:25.870065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519187999839172:2049] [node 2] ICS07 socket disconnect 86 reason# EndOfStream
2025-11-11T17:21:25.870103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519187999839172:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.870192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:25.870236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519187999839172:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:25.871230Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Couldn't connect to any resolved address
2025-11-11T17:21:25.871511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519187999839077:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:25.871575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519187999839172:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:25.871610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519187999839077:2049] [node 2] ICP30 unregister session Session# [4:7571519187999839172:2049] VirtualId# [4:7571519187999839166:0]
2025-11-11T17:21:25.871647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:25.871684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519187999839077:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:25.871733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:25.885698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519187999839077:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:25.901362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:25.901620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519187999839168:2050] [node 3] ICS01 socket: 84 reason# EndOfStream
2025-11-11T17:21:25.901659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519187999839078:2050] [node 3] ICP30 unregister session Session# [4:7571519187999839168:2050] VirtualId# [4:7571519187999839162:0]
2025-11-11T17:21:25.901697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519187999839168:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.901764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:25.901802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:25.902675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:25.902845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519185390428620:2050] [node 3] ICS01 socket: 55 reason# EndOfStream
2025-11-11T17:21:25.902878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519185390428532:2050] [node 3] ICP30 unregister session Session# [5:7571519185390428620:2050] VirtualId# [5:7571519185390428613:0]
2025-11-11T17:21:25.902908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519185390428620:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.902966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:25.903000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:25.906406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:25.906617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519186024993266:2050] [node 3] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-11T17:21:25.906651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519186024993266:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.906736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:25.906780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519186024993266:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:25.908297Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24401) Couldn't connect to any resolved address
2025-11-11T17:21:25.915901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519186024993180:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24401) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:25.915994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519186024993266:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:25.916045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519186024993180:2050] [node 3] ICP30 unregister session Session# [6:7571519186024993266:2050] VirtualId# [6:7571519186024993263:0]
2025-11-11T17:21:25.916103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:25.916139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519186024993180:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24401) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:25.916157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24401) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:25.927056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519186024993180:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:25.965769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:25.966021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519186024993281:2109] [node 4] ICS01 socket: 98 reason# EndOfStream
2025-11-11T17:21:25.966070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519186024993244:2109] [node 4] ICP30 unregister session Session# [6:7571519186024993281:2109] VirtualId# [6:7571519186024993279:0]
2025-11-11T17:21:25.966106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519186024993281:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.966208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:25.966261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:25.969107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:25.969477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519185390428634:2108] [node 4] ICS01 socket: 91 reason# EndOfStream
2025-11-11T17:21:25.969543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519185390428594:2108] [node 4] ICP30 unregister session Session# [5:7571519185390428634:2108] VirtualId# [5:7571519185390428629:0]
2025-11-11T17:21:25.969633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519185390428634:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:25.969799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:25.969891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:26.002828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:26.003040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519186024993275:2106] [node 5] ICS01 socket: 42 reason# EndOfStream
2025-11-11T17:21:26.003091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519186024993237:2106] [node 5] ICP30 unregister session Session# [6:7571519186024993275:2106] VirtualId# [6:7571519186024993272:0]
2025-11-11T17:21:26.003124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519186024993275:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:26.003207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:26.003251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:26.096021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519188592600075:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:26.096077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519188592600076:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:26.118540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519188272568002:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:26.118865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519188272568003:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:26.141378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519191874751360:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:26.141416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519191874751361:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:26.162839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519191922861500:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:26.162888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519191922861501:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:26.169742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519191922861502:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:26.186207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519189271731801:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:26.186249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519189271731802:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:26.186278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519189271731803:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:26.207964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519189003493425:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:26.208019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519189003493426:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:26.208069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519189003493427:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:26.270175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519189003493487:2106] [node 5] ICP01 ready to work
2025-11-11T17:21:26.291895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519188592600138:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:26.305564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519188592600139:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:26.384772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519188272568066:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:26.386655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519188592600075:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(38049872ns) {Configure:104(38044863ns) {ProcessConfigured:135(28189896ns) {StartInitialHandshake:157(28187125ns) {DropHandshakes:469(2659ns) {DropIncomingHandshake:441(576ns) DropOutgoingHandshake:456(462ns)} GenerateSessionVirtualId:988(497ns) PrepareNewSessionHandshake:476(11143ns) {SwitchToState:178(1307ns)}}}}}
2025-11-11T17:21:26.386226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519191874751423:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:26.391960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519188592600142:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:26.425706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519188272568067:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:26.404453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519191874751424:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:26.442698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519188272568068:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:26.414816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519191874751425:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:26.443501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519189271731865:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:26.443088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519191922861565:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:26.443350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519189003493492:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:26.446109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519191922861566:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:26.448028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519189271731866:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:26.448224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519189003493425:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(59634980ns) {Configure:104(59628456ns) {ProcessConfigured:135(2251052ns) {StartInitialHandshake:157(2248094ns) {DropHandshakes:469(2582ns) {DropIncomingHandshake:441(553ns) DropOutgoingHandshake:456(476ns)} GenerateSessionVirtualId:988(538ns) PrepareNewSessionHandshake:476(2159ns) {SwitchToState:178(703ns)}}}}}
2025-11-11T17:21:26.449752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519188272568002:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(148358400ns) {Configure:104(148353919ns) {ProcessConfigured:135(875619ns) {StartInitialHandshake:157(873731ns) {DropHandshakes:469(1343ns) {DropIncomingHandshake:441(322ns) DropOutgoingHandshake:456(253ns)} GenerateSessionVirtualId:988(455ns) PrepareNewSessionHandshake:476(1985ns) {SwitchToState:178(519ns)}}}}}
2025-11-11T17:21:26.467415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519188272568067:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17169693ns) {Configure:104(17165733ns) {ProcessConfigured:135(2619057ns) {StartInitialHandshake:157(2617178ns) {DropHandshakes:469(1350ns) {DropIncomingHandshake:441(355ns) DropOutgoingHandshake:456(135ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:476(2745ns) {SwitchToState:178(1008ns)}}}}}
2025-11-11T17:21:26.467419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519191922861565:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17296773ns) {Configure:104(17293959ns) {ProcessConfigured:135(6291969ns) {StartInitialHandshake:157(6290368ns) {DropHandshakes:469(1047135ns) {DropIncomingHandshake:441(1110ns) DropOutgoingHandshake:456(453ns)} GenerateSessionVirtualId:988(1179ns) PrepareNewSessionHandshake:476(3136ns) {SwitchToState:178(1635ns)}}}}}
2025-11-11T17:21:26.469738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519188592600138:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25820857ns) {Configure:104(25815426ns) {ProcessConfigured:135(271153ns) {StartInitialHandshake:157(269831ns) {DropHandshakes:469(908ns) {DropIncomingHandshake:441(255ns) DropOutgoingHandshake:456(194ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(1285ns) {SwitchToState:178(410ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.469844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519191874751360:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16295854ns) {Configure:104(16292355ns) {ProcessConfigured:135(76767ns) {StartInitialHandshake:157(76182ns) {DropHandshakes:469(1258ns) {DropIncomingHandshake:441(221ns) DropOutgoingHandshake:456(338ns)} GenerateSessionVirtualId:988(398ns) PrepareNewSessionHandshake:476(592ns) {SwitchToState:178(191ns)}}}}}
2025-11-11T17:21:26.471539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519189003493427:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18525904ns) {Configure:104(18518577ns) {ProcessConfigured:135(305603ns) {StartInitialHandshake:157(304417ns) {DropHandshakes:469(1178ns) {DropIncomingHandshake:441(345ns) DropOutgoingHandshake:456(158ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:476(981ns) {SwitchToState:178(424ns)}}}}}
2025-11-11T17:21:26.471822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519189271731801:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23566198ns) {Configure:104(23561595ns) {ProcessConfigured:135(1753530ns) {StartInitialHandshake:157(1752000ns) {DropHandshakes:469(746ns) {DropIncomingHandshake:441(167ns) DropOutgoingHandshake:456(98ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:476(2106ns) {SwitchToState:178(923ns)}}}}}
2025-11-11T17:21:26.470160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519188592600153:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61684
2025-11-11T17:21:26.472127Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600153:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.488375Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519191922861566:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28572624ns) {Configure:104(28560852ns) {ProcessConfigured:135(57207ns) {StartInitialHandshake:157(55218ns) {DropHandshakes:469(672ns) {DropIncomingHandshake:441(177ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:476(1898ns) {SwitchToState:178(516ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.502145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519188272568079:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8561
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.508234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519188272568080:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8561
2025-11-11T17:21:26.508801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568079:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.509000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568080:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.509364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519188272568002:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.509476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519188272568084:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8561
2025-11-11T17:21:26.501054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519189271731802:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(39568294ns) {Configure:104(39563441ns) {ProcessConfigured:135(1709387ns) {StartInitialHandshake:157(1706798ns) {DropHandshakes:469(400846ns) {DropIncomingHandshake:441(377ns) DropOutgoingHandshake:456(636ns)} GenerateSessionVirtualId:988(763ns) PrepareNewSessionHandshake:476(1909ns) {SwitchToState:178(605ns)}}}}}
2025-11-11T17:21:26.505910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519191874751424:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35612481ns) {Configure:104(35597849ns) {ProcessConfigured:135(7996689ns) {StartInitialHandshake:157(7993556ns) {DropHandshakes:469(2557ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(561ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:476(5877ns) {SwitchToState:178(2407ns)}}}}}
2025-11-11T17:21:26.513975Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600144:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.506789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519191874751436:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8214
2025-11-11T17:21:26.522362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493503:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.507360Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519191874751436:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.514024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519188592600144:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.521360Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600141:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.521391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519188592600141:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.507537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519191874751360:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.527203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731862:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.530460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519188272568087:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8561
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.534925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731875:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.535958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861578:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
2025-11-11T17:21:26.537330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519188592600158:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61684
2025-11-11T17:21:26.535482Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519189271731862:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.544617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519191874751441:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8214
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.544738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519191874751442:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8214
2025-11-11T17:21:26.536305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519188272568088:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8561
2025-11-11T17:21:26.543410Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519191922861578:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.545396Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519189003493503:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.537714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519188592600156:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61684
2025-11-11T17:21:26.545757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519189003493427:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.546277Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519191874751431:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.546307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519191874751431:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.547364Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519191874751431:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:26.537885Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519188592600144:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:26.549026Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519189271731875:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.549151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519189271731801:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.549785Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519188272568079:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:26.540464Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519188592600141:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.560898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519188272568002:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.560966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:26.561017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519188272568002:2048] [node 1] ICP22 created new session: [2:7571519188272568090:2048]
2025-11-11T17:21:26.561064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519188272568090:2048] [node 1] ICS09 handshake done sender: [2:7571519188272568079:2116] self: [2:7571519188272568085:0] peer: [1:7571519188592600140:0] socket: 50 qp: -1
2025-11-11T17:21:26.549053Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600158:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.558011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519191874751443:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8214
2025-11-11T17:21:26.561098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519188272568090:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:26.549158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519188592600075:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.572016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861579:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
2025-11-11T17:21:26.549209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:26.561308Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568084:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.570641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493505:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.571215Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519191874751441:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.549257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519188592600075:2048] [node 2] ICP22 created new session: [1:7571519188592600162:2048]
2025-11-11T17:21:26.549298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519188592600162:2048] [node 2] ICS09 handshake done sender: [1:7571519188592600141:2110] self: [1:7571519188592600140:0] peer: [2:7571519188272568085:0] socket: 44 qp: -1
2025-11-11T17:21:26.549336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519188592600162:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:26.549371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:26.552004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519188592600076:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.571968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519191874751425:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.572432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519191922861501:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.564594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:26.573568Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519191922861579:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.580948Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519189003493503:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:26.552048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:26.572017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:26.557825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519188592600076:2049] [node 3] ICP22 created new session: [1:7571519188592600164:2049]
2025-11-11T17:21:26.572413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519191874751425:2109] [node 6] ICP22 created new session: [3:7571519191874751445:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.590046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861582:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
2025-11-11T17:21:26.561327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519188592600164:2049] [node 3] ICS09 handshake done sender: [1:7571519188592600144:2111] self: [1:7571519188592600143:0] peer: [3:7571519191874751440:0] socket: 45 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.587039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519189003493427:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.587217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:26.587753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519189003493427:2050] [node 3] ICP22 created new session: [6:7571519189003493506:2050]
2025-11-11T17:21:26.587818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519189003493506:2050] [node 3] ICS09 handshake done sender: [6:7571519189003493503:2115] self: [6:7571519189003493504:0] peer: [3:7571519191874751430:0] socket: 57 qp: -1
2025-11-11T17:21:26.561368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519188592600164:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:26.572752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519191874751436:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:26.587874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519189003493506:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:26.601956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519191922861582:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.601699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519188272568089:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8561
2025-11-11T17:21:26.573525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519191874751445:2109] [node 6] ICS09 handshake done sender: [3:7571519191874751431:2112] self: [3:7571519191874751430:0] peer: [6:7571519189003493504:0] socket: 55 qp: -1
2025-11-11T17:21:26.565793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519188592600157:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61684
2025-11-11T17:21:26.573724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519191874751445:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:26.585716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:26.591381Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519191874751442:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.591445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519191874751360:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.591488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:26.591541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519191874751360:2048] [node 1] ICP22 created new session: [3:7571519191874751447:2048]
2025-11-11T17:21:26.591584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519191874751447:2048] [node 1] ICS09 handshake done sender: [3:7571519191874751436:2115] self: [3:7571519191874751440:0] peer: [1:7571519188592600143:0] socket: 53 qp: -1
2025-11-11T17:21:26.591616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519191874751447:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:26.591647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:26.592846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519191874751361:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.587996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.567579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:26.615360Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568074:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.624658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519191874751425:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(51864713ns) {HandleHandshakeStatus:329(51860973ns) {DropIncomingHandshake:441(538ns) DropOutgoingHandshake:456(716ns) SwitchToState:178(498ns) ProcessPendingSessionEvents:495(36538920ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36440105ns max# 11351947ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.587309Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600146:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.633016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493509:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.615427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519188272568074:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.587339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519188592600146:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.616962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600157:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.617082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600156:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.617345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519188592600075:2048] [node 2] ICP18 handshake (actor [1:7571519188592600158:2118]) is too old
2025-11-11T17:21:26.617648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519188592600161:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61684
2025-11-11T17:21:26.622466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519188592600076:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(60068252ns) {HandleHandshakeStatus:329(60064754ns) {DropIncomingHandshake:441(351ns) DropOutgoingHandshake:456(428ns) SwitchToState:178(588ns) ProcessPendingSessionEvents:495(40038426ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39012856ns max# 11651940ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.666078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861585:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
2025-11-11T17:21:26.636115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731884:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.658892Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568089:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.659009Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568087:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.659110Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568071:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.659133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519188272568071:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.675866Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519188272568074:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:26.675894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519188592600146:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.693998Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519188272568071:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:26.694326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519188272568066:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.682096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731883:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.682446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731885:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.696679Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519189271731875:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:26.684455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519191874751453:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8214
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.696111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519188272568002:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(40922789ns) {HandleHandshakeStatus:329(40918527ns) {DropIncomingHandshake:441(818ns) DropOutgoingHandshake:456(6050ns) SwitchToState:178(402ns) ProcessPendingSessionEvents:495(29579693ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29484956ns max# 10854555ns}}}
2025-11-11T17:21:26.697806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861584:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.698256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519189271731801:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.698201Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519191922861578:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:26.696197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519188272568003:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.702727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:26.710171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519191874751441:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:26.710539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519188592600076:2049] [node 3] ICP18 handshake (actor [1:7571519188592600156:2116]) is too old
2025-11-11T17:21:26.696241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:26.696277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519188272568003:2049] [node 3] ICP22 created new session: [2:7571519188272568098:2049]
2025-11-11T17:21:26.696313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519188272568098:2049] [node 3] ICS09 handshake done sender: [2:7571519188272568071:2111] self: [2:7571519188272568069:0] peer: [3:7571519191874751450:0] socket: 60 qp: -1
2025-11-11T17:21:26.696340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519188272568098:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:26.696365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:26.698427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:26.698466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519188272568066:2107] [node 4] ICP22 created new session: [2:7571519188272568100:2107]
2025-11-11T17:21:26.698509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519188272568100:2107] [node 4] ICS09 handshake done sender: [2:7571519188272568074:2113] self: [2:7571519188272568073:0] peer: [4:7571519191922861580:0] socket: 63 qp: -1
2025-11-11T17:21:26.698539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519188272568100:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:26.698572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:26.746260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519191874751361:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.714583Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600161:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.701706Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:26.705179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519189271731801:2048] [node 1] ICP22 created new session: [5:7571519189271731886:2048]
2025-11-11T17:21:26.753804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519188592600139:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.746351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:26.704453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
2025-11-11T17:21:26.714694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519189271731886:2048] [node 1] ICS09 handshake done sender: [5:7571519189271731875:2115] self: [5:7571519189271731882:0] peer: [1:7571519188592600145:0] socket: 52 qp: -1
2025-11-11T17:21:26.746420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519191874751361:2049] [node 2] ICP22 created new session: [3:7571519191874751454:2049]
2025-11-11T17:21:26.753867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.704834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519191922861501:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.784247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493511:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.714751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519189271731886:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:26.753925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519188592600139:2108] [node 5] ICP22 created new session: [1:7571519188592600170:2108]
2025-11-11T17:21:26.792607Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568076:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.792643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519188272568076:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.746515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519191874751454:2049] [node 2] ICS09 handshake done sender: [3:7571519191874751441:2116] self: [3:7571519191874751450:0] peer: [2:7571519188272568069:0] socket: 66 qp: -1
2025-11-11T17:21:26.724935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:26.801806Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519188272568076:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:26.802250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519188272568067:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.802320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:26.802377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519188272568067:2109] [node 5] ICP22 created new session: [2:7571519188272568104:2109]
2025-11-11T17:21:26.802423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519188272568104:2109] [node 5] ICS09 handshake done sender: [2:7571519188272568076:2114] self: [2:7571519188272568075:0] peer: [5:7571519189271731891:0] socket: 92 qp: -1
2025-11-11T17:21:26.802462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519188272568104:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:26.802503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:26.704884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:26.746558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519191874751454:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:26.753973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519188592600170:2108] [node 5] ICS09 handshake done sender: [1:7571519188592600146:2112] self: [1:7571519188592600145:0] peer: [5:7571519189271731882:0] socket: 46 qp: -1
2025-11-11T17:21:26.708326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519191922861501:2049] [node 2] ICP22 created new session: [4:7571519191922861588:2049]
2025-11-11T17:21:26.714409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519191922861588:2049] [node 2] ICS09 handshake done sender: [4:7571519191922861578:2115] self: [4:7571519191922861580:0] peer: [2:7571519188272568073:0] socket: 69 qp: -1
2025-11-11T17:21:26.723826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519191922861588:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:26.741870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:26.753195Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519191922861586:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.762951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519191922861502:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.771287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519191922861501:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(65579774ns) {HandleHandshakeStatus:329(65575522ns) {DropIncomingHandshake:441(511ns) DropOutgoingHandshake:456(6740ns) SwitchToState:178(1100ns) ProcessPendingSessionEvents:495(555197ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 491296ns max# 63532ns}}}
2025-11-11T17:21:26.771384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519191922861501:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:26.777410Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519191922861584:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.777525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519191922861500:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.790957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861592:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
2025-11-11T17:21:26.813879Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.826840Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519188272568078:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.826880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519188272568078:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.829134Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519188272568078:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:26.754012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519188592600170:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:26.780577Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519189271731884:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.797500Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519189003493511:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.829917Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519191922861584:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:26.746593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:26.834609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519188272568068:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.830485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519191922861502:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519191922861586:2120] held: no
2025-11-11T17:21:26.825606Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519189003493509:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.781078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731888:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.778994Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519191874751429:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.754048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:26.781208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519189271731803:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.781935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519189271731801:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(81616342ns) {HandleHandshakeStatus:329(81614409ns) {DropIncomingHandshake:441(600ns) DropOutgoingHandshake:456(7115ns) SwitchToState:178(2554ns) ProcessPendingSessionEvents:495(16979351ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16919975ns max# 15750410ns}}}
2025-11-11T17:21:26.783667Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519189271731885:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.783802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519189271731802:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.792067Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519189271731888:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.809914Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:26.823953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519189271731802:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519189271731885:2119] held: no
2025-11-11T17:21:26.824611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519189271731865:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.825192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.839130Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519189271731877:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.831240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519191922861572:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.839157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519189271731877:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.839784Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519189271731885:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:26.841869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519189271731802:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.834777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:26.825751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493510:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.775339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519188592600139:2108] [node 5] ICP18 handshake (actor [1:7571519188592600161:2119]) is too old
2025-11-11T17:21:26.779035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519191874751429:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.835355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519188272568068:2110] [node 6] ICP22 created new session: [2:7571519188272568107:2110]
2025-11-11T17:21:26.841913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:26.841965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519189271731802:2049] [node 2] ICP22 created new session: [5:7571519189271731897:2049]
2025-11-11T17:21:26.842017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519189271731897:2049] [node 2] ICS09 handshake done sender: [5:7571519189271731885:2119] self: [5:7571519189271731891:0] peer: [2:7571519188272568075:0] socket: 94 qp: -1
2025-11-11T17:21:26.842052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519189271731897:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:26.842083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:26.831267Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519191922861572:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.825951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519189003493425:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.795266Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519191874751429:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:26.826437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519189003493510:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.795540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519191874751423:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.826532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519189003493426:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.795597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:26.828107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493514:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.835174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519189003493487:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.835797Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519189003493510:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:26.843241Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:26.851256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519189003493426:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.778664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600154:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.845089Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:26.836605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519188272568107:2110] [node 6] ICS09 handshake done sender: [2:7571519188272568078:2115] self: [2:7571519188272568077:0] peer: [6:7571519189003493513:0] socket: 54 qp: -1
2025-11-11T17:21:26.795646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519191874751423:2107] [node 4] ICP22 created new session: [3:7571519191874751457:2107]
2025-11-11T17:21:26.851259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731895:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.845418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519191922861500:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.795695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519191874751457:2107] [node 4] ICS09 handshake done sender: [3:7571519191874751429:2111] self: [3:7571519191874751428:0] peer: [4:7571519191922861590:0] socket: 96 qp: -1
2025-11-11T17:21:26.852430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:26.778699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519188592600154:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.836643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519188272568107:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:26.795729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519191874751457:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:26.795761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:26.814526Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519191874751435:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.814571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519191874751435:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.825389Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519191874751435:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:26.836054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519191874751424:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.837187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:26.840828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519191874751424:2108] [node 5] ICP22 created new session: [3:7571519191874751460:2108]
2025-11-11T17:21:26.845497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:26.857911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519189003493426:2049] [node 2] ICP22 created new session: [6:7571519189003493516:2049]
2025-11-11T17:21:26.842487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519191874751460:2108] [node 5] ICS09 handshake done sender: [3:7571519191874751435:2114] self: [3:7571519191874751434:0] peer: [5:7571519189271731890:0] socket: 87 qp: -1
2025-11-11T17:21:26.845015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519191874751460:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:26.847673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:26.851887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519191874751462:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8214
2025-11-11T17:21:26.791611Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519188592600154:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:26.876012Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519189271731877:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.820063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519188592600138:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.845548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519191922861500:2048] [node 1] ICP22 created new session: [4:7571519191922861594:2048]
2025-11-11T17:21:26.836719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:26.868740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519189003493516:2049] [node 2] ICS09 handshake done sender: [6:7571519189003493510:2118] self: [6:7571519189003493513:0] peer: [2:7571519188272568077:0] socket: 58 qp: -1
2025-11-11T17:21:26.845615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519191922861594:2048] [node 1] ICS09 handshake done sender: [4:7571519191922861584:2118] self: [4:7571519191922861591:0] peer: [1:7571519188592600152:0] socket: 90 qp: -1
2025-11-11T17:21:26.855159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519188272568068:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(19898225ns) {HandleHandshakeStatus:329(19894892ns) {DropIncomingHandshake:441(24133ns) DropOutgoingHandshake:456(581ns) SwitchToState:178(492ns) ProcessPendingSessionEvents:495(17642810ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16827201ns max# 3631328ns}}}
2025-11-11T17:21:26.886612Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16290) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:26.871054Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:26.820142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.903065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519191874751424:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(66138768ns) {HandleHandshakeStatus:329(66135727ns) {DropIncomingHandshake:441(593ns) DropOutgoingHandshake:456(965ns) SwitchToState:178(2940ns) ProcessPendingSessionEvents:495(50670440ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 50580381ns max# 17213601ns}}}
2025-11-11T17:21:26.845678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519191922861594:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:26.911344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519189271731866:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.872533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519189003493516:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:26.876154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:26.878328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519189003493425:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519189003493509:2117] held: no
2025-11-11T17:21:26.879927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519189003493426:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(26457061ns) {HandleHandshakeStatus:329(26454507ns) {DropIncomingHandshake:441(578ns) DropOutgoingHandshake:456(4696ns) SwitchToState:178(607ns) ProcessPendingSessionEvents:495(1443721ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1379116ns max# 811053ns}}}
2025-11-11T17:21:26.880295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519189003493426:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:26.883737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493517:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.910534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519189003493425:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(31398972ns) {HandleHandshakeStatus:395(31391870ns) {LogHandshakeFail:472(15854ns) {UpdateErrorStateLog:544(3800ns)} DropOutgoingHandshake:456(465ns) ProcessPendingSessionEvents:495(31320257ns) {unordered DropSessionEvent:512 num# 100 sum# 31276461ns max# 8745552ns}}}
2025-11-11T17:21:26.910673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493518:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.823095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519188592600138:2107] [node 4] ICP22 created new session: [1:7571519188592600173:2107]
2025-11-11T17:21:26.833997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519188592600173:2107] [node 4] ICS09 handshake done sender: [1:7571519188592600154:2115] self: [1:7571519188592600152:0] peer: [4:7571519191922861591:0] socket: 86 qp: -1
2025-11-11T17:21:26.837132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519188592600173:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:26.840654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:26.840710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519188592600148:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.840740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519188592600148:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.845263Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519188592600148:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:26.851537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519188592600142:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.852228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:26.852277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519188592600142:2109] [node 6] ICP22 created new session: [1:7571519188592600176:2109]
2025-11-11T17:21:26.863082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519188592600138:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(47393649ns) {HandleHandshakeStatus:329(47389913ns) {DropIncomingHandshake:441(20355ns) DropOutgoingHandshake:456(621ns) SwitchToState:178(412ns) ProcessPendingSessionEvents:495(18979709ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18906983ns max# 8579177ns}}}
2025-11-11T17:21:26.875141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519188592600176:2109] [node 6] ICS09 handshake done sender: [1:7571519188592600148:2113] self: [1:7571519188592600147:0] peer: [6:7571519189003493512:0] socket: 85 qp: -1
2025-11-11T17:21:26.876787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519188592600176:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:26.876834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.845713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:26.933215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493519:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.930580Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519191874751462:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.911466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:26.911537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519189271731866:2110] [node 6] ICP22 created new session: [5:7571519189271731900:2110]
2025-11-11T17:21:26.911582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519189271731900:2110] [node 6] ICS09 handshake done sender: [5:7571519189271731877:2116] self: [5:7571519189271731876:0] peer: [6:7571519189003493515:0] socket: 71 qp: -1
2025-11-11T17:21:26.911620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519189271731900:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:26.911680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:26.912869Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519189271731884:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:26.917849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519189271731865:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16290) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519189271731888:2120] held: no
2025-11-11T17:21:26.923086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519189271731803:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.923146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:26.923213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519189271731803:2050] [node 3] ICP22 created new session: [5:7571519189271731902:2050]
2025-11-11T17:21:26.923260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519189271731902:2050] [node 3] ICS09 handshake done sender: [5:7571519189271731884:2118] self: [5:7571519189271731890:0] peer: [3:7571519191874751434:0] socket: 68 qp: -1
2025-11-11T17:21:26.923301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519189271731902:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:26.923336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.950698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519189271731899:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64523
2025-11-11T17:21:26.942018Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:26.953720Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519189003493511:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:26.954121Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519189003493519:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.954176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519189003493509:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:26.846526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519191922861500:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:26.951926Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519189271731888:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:26.950465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519191874751424:2108] [node 5] ICP18 handshake (actor [3:7571519191874751462:2120]) is too old
2025-11-11T17:21:26.962034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519189271731865:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.961247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519188592600142:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(100135823ns) {HandleHandshakeStatus:329(99768129ns) {DropIncomingHandshake:441(12759ns) DropOutgoingHandshake:456(210ns) SwitchToState:178(981ns) ProcessPendingSessionEvents:495(60913328ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60813261ns max# 19089459ns}}}
2025-11-11T17:21:26.847123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519191922861593:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41535
2025-11-11T17:21:26.965796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519189003493487:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519189003493511:2119] held: no
2025-11-11T17:21:26.966401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519189003493487:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.966445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:26.966494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519189003493487:2106] [node 5] ICP22 created new session: [6:7571519189003493521:2106]
2025-11-11T17:21:26.966546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519189003493521:2106] [node 5] ICS09 handshake done sender: [6:7571519189003493511:2119] self: [6:7571519189003493515:0] peer: [5:7571519189271731876:0] socket: 75 qp: -1
2025-11-11T17:21:26.966594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519189003493521:2106] [node 5] ICS10 traffic start
2025-11-11T17:21:26.966669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:26.967613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519189003493425:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.967681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:26.967727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519189003493425:2048] [node 1] ICP22 created new session: [6:7571519189003493523:2048]
2025-11-11T17:21:26.967770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519189003493523:2048] [node 1] ICS09 handshake done sender: [6:7571519189003493509:2117] self: [6:7571519189003493512:0] peer: [1:7571519188592600147:0] socket: 89 qp: -1
2025-11-11T17:21:26.967806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519189003493523:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:26.967831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:26.968864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519189003493492:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:26.962119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:26.850520Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519191922861586:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:26.851500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519191922861502:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.852108Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519191922861572:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:26.852303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:26.852347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519191922861502:2050] [node 3] ICP22 created new session: [4:7571519191922861598:2050]
2025-11-11T17:21:26.852649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519191922861598:2050] [node 3] ICS09 handshake done sender: [4:7571519191922861586:2120] self: [4:7571519191922861590:0] peer: [3:7571519191874751428:0] socket: 98 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:26.852686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519191922861598:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:26.852910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:26.873004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519191922861565:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.873095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:26.873145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519191922861565:2109] [node 5] ICP22 created new session: [4:7571519191922861600:2109]
2025-11-11T17:21:26.873189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519191922861600:2109] [node 5] ICS09 handshake done sender: [4:7571519191922861572:2113] self: [4:7571519191922861571:0] peer: [5:7571519189271731893:0] socket: 65 qp: -1
2025-11-11T17:21:26.873231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519191922861600:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:26.873265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:26.876705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519191922861502:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(21721193ns) {HandleHandshakeStatus:329(21718964ns) {DropIncomingHandshake:441(503ns) DropOutgoingHandshake:456(170ns) SwitchToState:178(226ns) ProcessPendingSessionEvents:495(1216ns)}}
2025-11-11T17:21:26.969252Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519191922861574:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:26.969276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519191922861574:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:26.977916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519191922861574:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:26.982469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519189003493526:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-11T17:21:26.986916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519189003493519:2123] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:26.962184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519189271731865:2109] [node 4] ICP22 created new session: [5:7571519189271731904:2109]
2025-11-11T17:21:26.977878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519189271731904:2109] [node 4] ICS09 handshake done sender: [5:7571519189271731888:2120] self: [5:7571519189271731893:0] peer: [4:7571519191922861571:0] socket: 64 qp: -1
2025-11-11T17:21:26.977941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519189271731904:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:26.978038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:26.982389Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519189271731865:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(20251009ns) {HandleHandshakeStatus:329(20247822ns) {DropIncomingHandshake:441(561ns) DropOutgoingHandshake:456(356ns) SwitchToState:178(526ns) ProcessPendingSessionEvents:495(1746ns)}}
2025-11-11T17:21:26.988476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519189003493492:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:26.988534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:26.988580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519189003493492:2110] [node 4] ICP22 created new session: [6:7571519189003493527:2110]
2025-11-11T17:21:26.988631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519189003493527:2110] [node 4] ICS09 handshake done sender: [6:7571519189003493519:2123] self: [6:7571519189003493525:0] peer: [4:7571519191922861573:0] socket: 59 qp: -1
2025-11-11T17:21:26.988671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519189003493527:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:26.988711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:26.993861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519191922861566:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:26.994402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:26.997818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519191922861566:2110] [node 6] ICP22 created new session: [4:7571519191922861603:2110]
2025-11-11T17:21:26.997892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519191922861603:2110] [node 6] ICS09 handshake done sender: [4:7571519191922861574:2114] self: [4:7571519191922861573:0] peer: [6:7571519189003493525:0] socket: 107 qp: -1
2025-11-11T17:21:26.997944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519191922861603:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:26.998050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:27.023606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:27.023886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519191874751447:2048] [node 1] ICS01 socket: 53 reason# EndOfStream
2025-11-11T17:21:27.023998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519191874751360:2048] [node 1] ICP30 unregister session Session# [3:7571519191874751447:2048] VirtualId# [3:7571519191874751440:0]
2025-11-11T17:21:27.024077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519191874751447:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.024195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:27.024250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:27.026553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:27.026786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519191922861594:2048] [node 1] ICS01 socket: 90 reason# EndOfStream
2025-11-11T17:21:27.026831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519191922861500:2048] [node 1] ICP30 unregister session Session# [4:7571519191922861594:2048] VirtualId# [4:7571519191922861591:0]
2025-11-11T17:21:27.026869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519191922861594:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.026983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:27.027034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:27.029713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:27.029713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:27.037926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519189271731886:2048] [node 1] ICS07 socket disconnect 52 reason# EndOfStream
2025-11-11T17:21:27.037981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519189271731886:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.038110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:27.038167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519189271731886:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:27.040412Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address
2025-11-11T17:21:27.040793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519189271731801:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:27.040839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519189271731886:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:27.040868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519189271731801:2048] [node 1] ICP30 unregister session Session# [5:7571519189271731886:2048] VirtualId# [5:7571519189271731882:0]
2025-11-11T17:21:27.041015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:27.041055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519189271731801:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.041071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.037998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519188272568090:2048] [node 1] ICS07 socket disconnect 50 reason# EndOfStream
2025-11-11T17:21:27.038066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519188272568090:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.038221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:27.038291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519188272568090:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:27.041734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:27.041980Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address
2025-11-11T17:21:27.042286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519188272568002:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:27.042332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519188272568090:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:27.042361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519188272568002:2048] [node 1] ICP30 unregister session Session# [2:7571519188272568090:2048] VirtualId# [2:7571519188272568085:0]
2025-11-11T17:21:27.042705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:27.042749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571519188272568002:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.042762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.041933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519189003493523:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-11-11T17:21:27.045710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519189003493425:2048] [node 1] ICP30 unregister session Session# [6:7571519189003493523:2048] VirtualId# [6:7571519189003493512:0]
2025-11-11T17:21:27.045762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519189003493523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.049304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:27.049442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:27.054948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519189271731801:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:27.057691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571519188272568002:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:27.102040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:27.102327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519191922861588:2049] [node 2] ICS01 socket: 69 reason# EndOfStream
2025-11-11T17:21:27.102369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519191922861501:2049] [node 2] ICP30 unregister session Session# [4:7571519191922861588:2049] VirtualId# [4:7571519191922861580:0]
2025-11-11T17:21:27.102413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519191922861588:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.102483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:27.102553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:27.102091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:27.102342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519191874751454:2049] [node 2] ICS01 socket: 66 reason# EndOfStream
2025-11-11T17:21:27.102395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519191874751361:2049] [node 2] ICP30 unregister session Session# [3:7571519191874751454:2049] VirtualId# [3:7571519191874751450:0]
2025-11-11T17:21:27.102428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519191874751454:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.102576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:27.102632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:27.107368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:27.107695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519189271731897:2049] [node 2] ICS01 socket: 94 reason# EndOfStream
2025-11-11T17:21:27.107749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519189271731802:2049] [node 2] ICP30 unregister session Session# [5:7571519189271731897:2049] VirtualId# [5:7571519189271731891:0]
2025-11-11T17:21:27.107807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519189271731897:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.107909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:27.107957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:27.110579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:27.110863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519189003493516:2049] [node 2] ICS07 socket disconnect 58 reason# EndOfStream
2025-11-11T17:21:27.110906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519189003493516:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.122348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:27.122479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519189003493516:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:27.128542Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Couldn't connect to any resolved address
2025-11-11T17:21:27.128901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519189003493426:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:27.128961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519189003493516:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:27.128997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519189003493426:2049] [node 2] ICP30 unregister session Session# [6:7571519189003493516:2049] VirtualId# [6:7571519189003493513:0]
2025-11-11T17:21:27.129105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:27.129143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519189003493426:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.129158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28961) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.139375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519189003493426:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:27.171982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:27.172206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519189271731902:2050] [node 3] ICS07 socket disconnect 68 reason# EndOfStream
2025-11-11T17:21:27.172234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519189271731902:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.172306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:27.172341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519189271731902:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:27.173008Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5664) Couldn't connect to any resolved address
2025-11-11T17:21:27.173090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519189271731803:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5664) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:27.173184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519189271731902:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:27.173234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519189271731803:2050] [node 3] ICP30 unregister session Session# [5:7571519189271731902:2050] VirtualId# [5:7571519189271731890:0]
2025-11-11T17:21:27.173278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:27.173316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519189271731803:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5664) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.173329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5664) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:27.173759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:27.173909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519191922861598:2050] [node 3] ICS01 socket: 98 reason# EndOfStream
2025-11-11T17:21:27.173935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519191922861502:2050] [node 3] ICP30 unregister session Session# [4:7571519191922861598:2050] VirtualId# [4:7571519191922861590:0]
2025-11-11T17:21:27.173962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519191922861598:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.173734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:27.174013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:27.174042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:27.173937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519189003493506:2050] [node 3] ICS01 socket: 57 reason# EndOfStream
2025-11-11T17:21:27.173967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519189003493427:2050] [node 3] ICP30 unregister session Session# [6:7571519189003493506:2050] VirtualId# [6:7571519189003493504:0]
2025-11-11T17:21:27.173999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519189003493506:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.174071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:27.174109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:27.183420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519189271731803:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:27.249759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:27.249989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519189003493527:2110] [node 4] ICS01 socket: 59 reason# EndOfStream
2025-11-11T17:21:27.250025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519189003493492:2110] [node 4] ICP30 unregister session Session# [6:7571519189003493527:2110] VirtualId# [6:7571519189003493525:0]
2025-11-11T17:21:27.250059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519189003493527:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.250152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:27.250197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:27.253746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:27.253980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519189271731904:2109] [node 4] ICS01 socket: 64 reason# EndOfStream
2025-11-11T17:21:27.254055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519189271731865:2109] [node 4] ICP30 unregister session Session# [5:7571519189271731904:2109] VirtualId# [5:7571519189271731893:0]
2025-11-11T17:21:27.254091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519189271731904:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.254209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:27.254294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:27.321745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:27.321961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519189003493521:2106] [node 5] ICS01 socket: 75 reason# EndOfStream
2025-11-11T17:21:27.322005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519189003493487:2106] [node 5] ICP30 unregister session Session# [6:7571519189003493521:2106] VirtualId# [6:7571519189003493515:0]
2025-11-11T17:21:27.322042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519189003493521:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:27.322137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:27.322186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:27.465144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519192644608616:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:27.465318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519192644608615:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:27.492515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519192989990464:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:27.492557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519192989990465:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:27.520546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519194962027804:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:27.520607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519194962027805:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:27.542849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519194698667797:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:27.542888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519194698667798:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:27.542916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519194698667799:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:27.591491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519196550732316:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:27.591532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519196550732317:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:27.591560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519196550732318:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:27.617690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519195335786429:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:27.617735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519195335786430:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:27.617764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519195335786431:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:27.846744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519192644608676:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:27.856193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519195335786486:2099] [node 4] ICP01 ready to work
2025-11-11T17:21:27.888307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519195335786431:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24163428ns) {Configure:104(24157190ns) {ProcessConfigured:135(100575ns) {StartInitialHandshake:157(97104ns) {DropHandshakes:469(3644ns) {DropIncomingHandshake:441(928ns) DropOutgoingHandshake:456(579ns)} GenerateSessionVirtualId:988(1238ns) PrepareNewSessionHandshake:476(2343ns) {SwitchToState:178(721ns)}}}}}
2025-11-11T17:21:27.918251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519192644608615:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(65886537ns) {Configure:104(65881686ns) {ProcessConfigured:135(6027269ns) {StartInitialHandshake:157(6025194ns) {DropHandshakes:469(1085ns) {DropIncomingHandshake:441(258ns) DropOutgoingHandshake:456(268ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:476(2859ns) {SwitchToState:178(493ns)}}}}}
2025-11-11T17:21:27.918672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519192644608683:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:27.923242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519192989990464:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(37089101ns) {Configure:104(37083888ns) {ProcessConfigured:135(63797ns) {StartInitialHandshake:157(61454ns) {DropHandshakes:469(2300ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(571ns)} GenerateSessionVirtualId:988(741ns) PrepareNewSessionHandshake:476(1444ns) {SwitchToState:178(382ns)}}}}}
2025-11-11T17:21:27.928468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519195335786496:2101] [node 5] ICP01 ready to work
2025-11-11T17:21:27.939885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519192989990528:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:27.940109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519192644608616:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(83283120ns) {Configure:104(83277324ns) {ProcessConfigured:135(6273395ns) {StartInitialHandshake:157(6269841ns) {DropHandshakes:469(2572ns) {DropIncomingHandshake:441(981ns) DropOutgoingHandshake:456(379ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:476(8566ns) {SwitchToState:178(3367ns)}}}}}
2025-11-11T17:21:27.940023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519194962027867:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:27.941910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519194962027868:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:27.957819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519194698667860:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:27.941416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519192989990531:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:27.945713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519192989990532:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:27.962211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519194962027869:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:27.982253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519192644608684:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:27.981958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519194698667861:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:27.987994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519192644608676:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(121996453ns) {Configure:104(121989082ns) {ProcessConfigured:135(20746283ns) {StartInitialHandshake:157(20743046ns) {DropHandshakes:469(7511978ns) {DropIncomingHandshake:441(1264ns) DropOutgoingHandshake:456(510ns)} GenerateSessionVirtualId:988(1441ns) PrepareNewSessionHandshake:476(5965ns) {SwitchToState:178(2710ns)}}}}}
2025-11-11T17:21:28.029616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519200845699677:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.058924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635167:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:21:28.061900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519192644608683:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(69289400ns) {Configure:104(69283110ns) {ProcessConfigured:135(28715469ns) {StartInitialHandshake:157(28712949ns) {DropHandshakes:469(3232ns) {DropIncomingHandshake:441(633ns) DropOutgoingHandshake:456(661ns)} GenerateSessionVirtualId:988(473ns) PrepareNewSessionHandshake:476(4155ns) {SwitchToState:178(1961ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.088803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519192989990528:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(26467055ns) {Configure:104(26460401ns) {ProcessConfigured:135(64222ns) {StartInitialHandshake:157(62286ns) {DropHandshakes:469(1416ns) {DropIncomingHandshake:441(402ns) DropOutgoingHandshake:456(223ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:476(1539ns) {SwitchToState:178(531ns)}}}}}
2025-11-11T17:21:28.087074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519196939575990:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4132
2025-11-11T17:21:28.089313Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519198993635167:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.098318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519200845699678:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:28.106437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519196550732317:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(151679311ns) {Configure:104(151671850ns) {ProcessConfigured:135(76648ns) {StartInitialHandshake:157(74180ns) {DropHandshakes:469(5229ns) {DropIncomingHandshake:441(574ns) DropOutgoingHandshake:456(300ns)} GenerateSessionVirtualId:988(1230ns) PrepareNewSessionHandshake:476(1694ns) {SwitchToState:178(505ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.112734Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519196939575990:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.112933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519196939575991:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4132
2025-11-11T17:21:28.092973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519195335786486:2099] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(156792290ns) {Configure:104(156779868ns) {ProcessConfigured:135(12203503ns) {StartInitialHandshake:157(12200220ns) {DropHandshakes:469(2358ns) {DropIncomingHandshake:441(425ns) DropOutgoingHandshake:456(368ns)} GenerateSessionVirtualId:988(586ns) PrepareNewSessionHandshake:476(3064ns) {SwitchToState:178(1021ns)}}}}}
2025-11-11T17:21:28.109882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519194698667797:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.112842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519194698667797:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18553980ns) {IncomingHandshake:261(16827282ns) {DropIncomingHandshake:441(266ns) PrepareNewSessionHandshake:476(7253ns) {SwitchToState:178(4819ns)} GenerateSessionVirtualId:988(976ns) IssueIncomingHandshakeReply:193(11967109ns) {SwitchToState:178(2445ns)}}}
2025-11-11T17:21:28.123839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519194698667798:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(35895015ns) {Configure:104(35889266ns) {ProcessConfigured:135(5997832ns) {StartInitialHandshake:157(5995583ns) {DropHandshakes:469(2725ns) {DropIncomingHandshake:441(537ns) DropOutgoingHandshake:456(543ns)} GenerateSessionVirtualId:988(670ns) PrepareNewSessionHandshake:476(2694ns) {SwitchToState:178(1054ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.129654Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519192644608686:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.129721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519192644608686:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.124248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519195335786496:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(52307417ns) {Configure:104(50145545ns) {ProcessConfigured:135(18903562ns) {StartInitialHandshake:157(18901946ns) {DropHandshakes:469(2102ns) {DropIncomingHandshake:441(836ns) DropOutgoingHandshake:456(345ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:476(2050ns) {SwitchToState:178(361ns)}}}}}
2025-11-11T17:21:28.146091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519194962027868:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(118002129ns) {Configure:104(117996102ns) {ProcessConfigured:135(97691ns) {StartInitialHandshake:157(93652ns) {DropHandshakes:469(4027ns) {DropIncomingHandshake:441(932ns) DropOutgoingHandshake:456(783ns)} GenerateSessionVirtualId:988(1122ns) PrepareNewSessionHandshake:476(3052ns) {SwitchToState:178(1388ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.134542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519192989990465:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(58875763ns) {Configure:104(58868776ns) {ProcessConfigured:135(10575782ns) {StartInitialHandshake:157(10573125ns) {DropHandshakes:469(1528ns) {DropIncomingHandshake:441(245ns) DropOutgoingHandshake:456(336ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:476(4269ns) {SwitchToState:178(1909ns)}}}}}
2025-11-11T17:21:28.133957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635173:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.149489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753795:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.174940Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519194698667863:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.174981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519194698667863:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.156690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519197284957832:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41584
2025-11-11T17:21:28.174137Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519192644608686:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.149683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753796:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
2025-11-11T17:21:28.183714Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519192644608682:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.184003Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519194698667863:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:28.171016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957832:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.166449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519195335786430:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(95250970ns) {Configure:104(95240529ns) {ProcessConfigured:135(74007ns) {StartInitialHandshake:157(65604ns) {DropHandshakes:469(2723ns) {DropIncomingHandshake:441(709ns) DropOutgoingHandshake:456(497ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:476(1472ns) {SwitchToState:178(461ns)}}}}}
2025-11-11T17:21:28.166612Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519199630753795:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.173783Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519199630753796:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.194194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519194962027869:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(127356577ns) {Configure:104(127061421ns) {ProcessConfigured:135(81956ns) {StartInitialHandshake:157(67959ns) {DropHandshakes:469(3983ns) {DropIncomingHandshake:441(382ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:476(1434ns) {SwitchToState:178(470ns)}}}}}
2025-11-11T17:21:28.174576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519195335786486:2099] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.183778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519192644608682:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.194601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519194962027870:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22539
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.171158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519197284957841:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41584
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.171347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519192989990464:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.189983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519192644608676:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.194790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519199256995167:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22539
2025-11-11T17:21:28.188841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753799:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.195229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519199256995174:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22539
2025-11-11T17:21:28.196088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753807:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
2025-11-11T17:21:28.198199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519197284957843:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41584
2025-11-11T17:21:28.211107Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519194962027870:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.210650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519195335786429:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53821186ns) {Configure:104(53815909ns) {ProcessConfigured:135(20068358ns) {StartInitialHandshake:157(20065488ns) {DropHandshakes:469(2449281ns) {DropIncomingHandshake:441(1603ns) DropOutgoingHandshake:456(862ns)} GenerateSessionVirtualId:988(1573ns) PrepareNewSessionHandshake:476(3343ns) {SwitchToState:178(1000ns)}}}}}
2025-11-11T17:21:28.206835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957841:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.217774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519199256995178:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22539
2025-11-11T17:21:28.211873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519199630753795:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:28.222968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519197284957846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41584
2025-11-11T17:21:28.212239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519195335786486:2099] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.223365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519197284957844:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41584
2025-11-11T17:21:28.212299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:28.212347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519195335786486:2099] [node 4] ICP22 created new session: [6:7571519199630753808:2099]
2025-11-11T17:21:28.212391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519199630753808:2099] [node 4] ICS09 handshake done sender: [6:7571519199630753795:2112] self: [6:7571519199630753805:0] peer: [4:7571519194698667862:0] socket: 58 qp: -1
2025-11-11T17:21:28.213932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519199630753808:2099] [node 4] ICS10 traffic start
2025-11-11T17:21:28.214906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:28.215886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519195335786429:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.190052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:28.221741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519196550732316:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(143980769ns) {Configure:104(143974059ns) {ProcessConfigured:135(17688899ns) {StartInitialHandshake:157(17685103ns) {DropHandshakes:469(2628ns) {DropIncomingHandshake:441(620ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(787ns) PrepareNewSessionHandshake:476(3526ns) {SwitchToState:178(1684ns)}}}}}
2025-11-11T17:21:28.190105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519192644608676:2107] [node 4] ICP22 created new session: [1:7571519196939575994:2107]
2025-11-11T17:21:28.224153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699688:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.222401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
2025-11-11T17:21:28.227619Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:28.232093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519199256995167:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.233126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519194962027804:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(138441716ns) {Configure:104(138431606ns) {ProcessConfigured:135(9650142ns) {StartInitialHandshake:157(9647081ns) {DropHandshakes:469(2070ns) {DropIncomingHandshake:441(358ns) DropOutgoingHandshake:456(423ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:476(247418ns) {SwitchToState:178(1839ns)}}}}}
2025-11-11T17:21:28.233141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519199256995174:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.226008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519199630753799:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.224235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699691:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.190153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519196939575994:2107] [node 4] ICS09 handshake done sender: [1:7571519192644608686:2112] self: [1:7571519192644608685:0] peer: [4:7571519198993635172:0] socket: 55 qp: -1
2025-11-11T17:21:28.227643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519194698667860:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.235985Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519199256995173:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.226177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519195335786496:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.230828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519199630753811:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.234255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519195335786431:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.238032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519197284957832:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:28.236037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519199256995173:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.227691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:28.227689Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519200845699684:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.190187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519196939575994:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.236140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519194962027867:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(70333456ns) {Configure:104(70327820ns) {ProcessConfigured:135(1846626ns) {StartInitialHandshake:157(1844204ns) {DropHandshakes:469(2615ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(538ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:476(2605ns) {SwitchToState:178(875ns)}}}}}
2025-11-11T17:21:28.240273Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957843:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.227741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519194698667860:2109] [node 6] ICP22 created new session: [4:7571519198993635178:2109]
2025-11-11T17:21:28.227732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519200845699684:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.255145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753815:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
2025-11-11T17:21:28.190225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:28.249740Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519199256995173:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-11T17:21:28.228000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519200845699691:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-11T17:21:28.245273Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957846:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.205675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519196939575991:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.272191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519199630753799:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:28.207298Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519192644608682:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:28.220869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519192644608615:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.220948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:28.223469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519192644608615:2048] [node 2] ICP22 created new session: [1:7571519196939575999:2048]
2025-11-11T17:21:28.225580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519196939575999:2048] [node 2] ICS09 handshake done sender: [1:7571519192644608682:2111] self: [1:7571519192644608680:0] peer: [2:7571519197284957842:0] socket: 54 qp: -1
2025-11-11T17:21:28.231613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519196939575999:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:28.234297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:28.241319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519192644608676:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(50579593ns) {HandleHandshakeStatus:329(50576256ns) {DropIncomingHandshake:441(16059ns) DropOutgoingHandshake:456(371ns) SwitchToState:178(362ns) ProcessPendingSessionEvents:495(50147748ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 50016371ns max# 6215868ns}}}
2025-11-11T17:21:28.241448Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519196939575985:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.241472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519196939575985:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.227787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519198993635178:2109] [node 6] ICS09 handshake done sender: [4:7571519194698667863:2110] self: [4:7571519194698667862:0] peer: [6:7571519199630753805:0] socket: 50 qp: -1
2025-11-11T17:21:28.227820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519198993635178:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:28.245417Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957834:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.230756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.238152Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519200845699688:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.277675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519199256995178:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.242642Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519200845699684:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:28.232775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519194698667797:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519198993635167:2113] held: no
2025-11-11T17:21:28.279170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519195335786496:2101] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.245437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519197284957834:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.243075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519196550732317:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.260890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519200845699677:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24373597ns) {Configure:104(24367354ns) {ProcessConfigured:135(70800ns) {StartInitialHandshake:157(61885ns) {DropHandshakes:469(2365ns) {DropIncomingHandshake:441(680ns) DropOutgoingHandshake:456(672ns)} GenerateSessionVirtualId:988(898ns) PrepareNewSessionHandshake:476(1459ns) {SwitchToState:178(558ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.278674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519194962027869:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.262208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519200845699678:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.273616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:28.247757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519192989990464:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.245023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635175:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:21:28.279848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:28.277947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519200845699678:2108] [node 6] ICP22 created new session: [5:7571519200845699697:2108]
2025-11-11T17:21:28.279698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519200845699697:2108] [node 6] ICS09 handshake done sender: [5:7571519200845699684:2114] self: [5:7571519200845699683:0] peer: [6:7571519199630753813:0] socket: 67 qp: -1
2025-11-11T17:21:28.279911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519195335786496:2101] [node 5] ICP22 created new session: [6:7571519199630753817:2101]
2025-11-11T17:21:28.280062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519200845699697:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:28.280577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:28.279968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519199630753817:2101] [node 5] ICS09 handshake done sender: [6:7571519199630753799:2115] self: [6:7571519199630753813:0] peer: [5:7571519200845699683:0] socket: 71 qp: -1
2025-11-11T17:21:28.280648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699698:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.280720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519199630753817:2101] [node 5] ICS10 traffic start
2025-11-11T17:21:28.289281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699699:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.280754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:28.279032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:28.293963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753816:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
2025-11-11T17:21:28.293163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519192644608615:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(66033934ns) {HandleHandshakeStatus:329(66030145ns) {DropIncomingHandshake:441(18107ns) DropOutgoingHandshake:456(800ns) SwitchToState:178(619ns) ProcessPendingSessionEvents:495(48675301ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 48570921ns max# 13189943ns}}}
2025-11-11T17:21:28.296103Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519192644608681:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.296127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519192644608681:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.298660Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519196939575985:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:28.254438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635177:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.247812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.247868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519192989990464:2048] [node 1] ICP22 created new session: [2:7571519197284957847:2048]
2025-11-11T17:21:28.247925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519197284957847:2048] [node 1] ICS09 handshake done sender: [2:7571519197284957832:2113] self: [2:7571519197284957842:0] peer: [1:7571519192644608680:0] socket: 59 qp: -1
2025-11-11T17:21:28.247957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519197284957847:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:28.247998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:28.277451Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519197284957834:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:28.293501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519192989990528:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.293564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:28.293632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519192989990528:2108] [node 5] ICP22 created new session: [2:7571519197284957852:2108]
2025-11-11T17:21:28.293674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519197284957852:2108] [node 5] ICS09 handshake done sender: [2:7571519197284957834:2114] self: [2:7571519197284957833:0] peer: [5:7571519200845699693:0] socket: 70 qp: -1
2025-11-11T17:21:28.293708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519197284957852:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:28.293742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:28.279742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519194962027869:2109] [node 6] ICP22 created new session: [3:7571519199256995187:2109]
2025-11-11T17:21:28.272989Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519198993635173:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.319621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519192989990528:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(25327295ns) {HandleHandshakeStatus:329(25325272ns) {DropIncomingHandshake:441(13579ns) DropOutgoingHandshake:456(435ns) SwitchToState:178(136ns) ProcessPendingSessionEvents:495(24978262ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24903764ns max# 6348681ns}}}
2025-11-11T17:21:28.319926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957830:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.319948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519197284957830:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.281912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519199256995187:2109] [node 6] ICS09 handshake done sender: [3:7571519199256995173:2114] self: [3:7571519199256995172:0] peer: [6:7571519199630753814:0] socket: 60 qp: -1
2025-11-11T17:21:28.308466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519196939576003:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4132
2025-11-11T17:21:28.273331Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519198993635167:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:28.278672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635176:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:21:28.280661Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:28.289541Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519198993635177:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.289613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519194698667797:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.289664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:28.289719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519194698667797:2048] [node 1] ICP22 created new session: [4:7571519198993635182:2048]
2025-11-11T17:21:28.289787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519198993635182:2048] [node 1] ICS09 handshake done sender: [4:7571519198993635167:2113] self: [4:7571519198993635172:0] peer: [1:7571519192644608685:0] socket: 56 qp: -1
2025-11-11T17:21:28.289820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519198993635182:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:28.289854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:28.290184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519194698667798:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.291794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519194698667799:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.308644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519196939576004:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4132
2025-11-11T17:21:28.283963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519194962027804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.308725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519192644608684:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.308768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:28.308813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519192644608684:2109] [node 6] ICP22 created new session: [1:7571519196939576006:2109]
2025-11-11T17:21:28.308861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519196939576006:2109] [node 6] ICS09 handshake done sender: [1:7571519196939575985:2113] self: [1:7571519196939575984:0] peer: [6:7571519199630753809:0] socket: 57 qp: -1
2025-11-11T17:21:28.308897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519196939576006:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:28.308930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:28.311698Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519192644608681:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:28.312089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519192644608616:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.312130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:28.312174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519192644608616:2049] [node 3] ICP22 created new session: [1:7571519196939576008:2049]
2025-11-11T17:21:28.312219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519196939576008:2049] [node 3] ICS09 handshake done sender: [1:7571519192644608681:2110] self: [1:7571519192644608679:0] peer: [3:7571519199256995188:0] socket: 88 qp: -1
2025-11-11T17:21:28.312250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519196939576008:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:28.312281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:28.321736Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519196939576004:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.321850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519192644608616:2049] [node 3] ICP18 handshake (actor [1:7571519196939576004:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.287937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519199256995187:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:28.337132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519194698667860:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(102951510ns) {HandleHandshakeStatus:329(102948974ns) {DropIncomingHandshake:441(482ns) DropOutgoingHandshake:456(437ns) SwitchToState:178(177ns) ProcessPendingSessionEvents:495(93861190ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 82079468ns max# 8252385ns}}}
2025-11-11T17:21:28.336413Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519199630753811:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.287979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.369510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519197284957854:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41584
2025-11-11T17:21:28.339486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519195335786431:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.367267Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:28.303277Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519199256995169:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.351286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:21:28.339547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:28.377938Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519197284957830:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:28.382456Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519198993635177:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:28.303319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519199256995169:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.390105Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519200845699688:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:28.339596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519195335786431:2050] [node 3] ICP22 created new session: [6:7571519199630753820:2050]
2025-11-11T17:21:28.394244Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957854:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.326009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519199256995192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22539
2025-11-11T17:21:28.384765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519194698667799:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.408533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519196550732317:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519200845699688:2115] held: no
2025-11-11T17:21:28.339667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519199630753820:2050] [node 3] ICS09 handshake done sender: [6:7571519199630753811:2119] self: [6:7571519199630753814:0] peer: [3:7571519199256995172:0] socket: 87 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.403008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519192989990531:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.327186Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519199256995169:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:28.384825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:28.417997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519200845699698:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.431735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519196939576010:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4132
2025-11-11T17:21:28.431810Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:28.338894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519194962027868:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.384888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519194698667799:2050] [node 3] ICP22 created new session: [4:7571519198993635188:2050]
2025-11-11T17:21:28.403057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:28.339703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519199630753820:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:28.424965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.339740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:28.384974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519198993635188:2050] [node 3] ICS09 handshake done sender: [4:7571519198993635177:2119] self: [4:7571519198993635185:0] peer: [3:7571519199256995168:0] socket: 82 qp: -1
2025-11-11T17:21:28.403109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519192989990531:2107] [node 4] ICP22 created new session: [2:7571519197284957858:2107]
2025-11-11T17:21:28.338970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:28.403157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519197284957858:2107] [node 4] ICS09 handshake done sender: [2:7571519197284957830:2112] self: [2:7571519197284957829:0] peer: [4:7571519198993635184:0] socket: 66 qp: -1
2025-11-11T17:21:28.354733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753819:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
2025-11-11T17:21:28.425159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519196550732317:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.385019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519198993635188:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:28.339019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519194962027868:2107] [node 4] ICP22 created new session: [3:7571519199256995193:2107]
2025-11-11T17:21:28.400064Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519199630753796:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:28.450721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519196939576003:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.455945Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519196939575989:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.455983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519196939575989:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.385055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:28.403208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519197284957858:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:28.426596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519200845699677:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.418535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519195335786429:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.339078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519199256995193:2107] [node 4] ICS09 handshake done sender: [3:7571519199256995169:2112] self: [3:7571519199256995168:0] peer: [4:7571519198993635185:0] socket: 81 qp: -1
2025-11-11T17:21:28.403248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:28.427511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:28.339114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519199256995193:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:28.387192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635187:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:21:28.418610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.418659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519195335786429:2048] [node 1] ICP22 created new session: [6:7571519199630753824:2048]
2025-11-11T17:21:28.418704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519199630753824:2048] [node 1] ICS09 handshake done sender: [6:7571519199630753796:2113] self: [6:7571519199630753809:0] peer: [1:7571519196939575984:0] socket: 61 qp: -1
2025-11-11T17:21:28.418740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519199630753824:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:28.418770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:28.437833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753822:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.451504Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519199630753822:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.452074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519195335786496:2101] [node 5] ICP35 event processing took too much time PendingConnection:278(116317060ns) {HandleHandshakeStatus:329(116314030ns) {DropIncomingHandshake:441(3581ns) DropOutgoingHandshake:456(8378ns) SwitchToState:178(342ns) ProcessPendingSessionEvents:495(114577268ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 109152326ns max# 10024686ns}}}
2025-11-11T17:21:28.454340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519195335786430:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.478631Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519196939575989:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:28.437705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519196550732317:2049] [node 2] ICP22 created new session: [5:7571519200845699704:2049]
2025-11-11T17:21:28.404079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519192989990531:2107] [node 4] ICP18 handshake (actor [2:7571519197284957854:2121]) is too old
2025-11-11T17:21:28.433869Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:28.339170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:28.473620Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957836:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.443581Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519198993635173:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:28.369795Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519199256995178:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:28.441794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519200845699704:2049] [node 2] ICS09 handshake done sender: [5:7571519200845699688:2115] self: [5:7571519200845699693:0] peer: [2:7571519197284957833:0] socket: 72 qp: -1
2025-11-11T17:21:28.457061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519194698667860:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:28.492724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519192644608683:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.473664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519197284957836:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.479475Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519197284957836:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:28.490602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519192989990532:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.497549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:28.499422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519192989990532:2110] [node 6] ICP22 created new session: [2:7571519197284957861:2110]
2025-11-11T17:21:28.504952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519197284957861:2110] [node 6] ICS09 handshake done sender: [2:7571519197284957836:2115] self: [2:7571519197284957835:0] peer: [6:7571519199630753826:0] socket: 63 qp: -1
2025-11-11T17:21:28.505019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519197284957861:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:28.505070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:28.441872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519200845699704:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:28.409405Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:28.461987Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519198993635169:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.511735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:28.518258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519192644608683:2108] [node 5] ICP22 created new session: [1:7571519196939576013:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.462035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519198993635169:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.465679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519194698667798:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519198993635173:2116] held: no
2025-11-11T17:21:28.519122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519199630753827:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39450
2025-11-11T17:21:28.476189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519194698667798:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.476255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:28.476288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519194698667798:2049] [node 2] ICP22 created new session: [4:7571519198993635191:2049]
2025-11-11T17:21:28.476326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519198993635191:2049] [node 2] ICS09 handshake done sender: [4:7571519198993635173:2116] self: [4:7571519198993635184:0] peer: [2:7571519197284957829:0] socket: 68 qp: -1
2025-11-11T17:21:28.476350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519198993635191:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:28.476376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:28.481994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519198993635190:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29282
2025-11-11T17:21:28.484363Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519198993635169:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:28.484412Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519198993635190:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.499218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519194698667861:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.499298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:28.506535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519194698667861:2108] [node 5] ICP22 created new session: [4:7571519198993635195:2108]
2025-11-11T17:21:28.510963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519198993635195:2108] [node 5] ICS09 handshake done sender: [4:7571519198993635169:2114] self: [4:7571519198993635168:0] peer: [5:7571519200845699705:0] socket: 95 qp: -1
2025-11-11T17:21:28.513198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519198993635195:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:28.513272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:28.441928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:28.442167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519196550732317:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16920206ns) {HandleHandshakeStatus:329(16917930ns) {DropIncomingHandshake:441(970ns) DropOutgoingHandshake:456(395ns) SwitchToState:178(1077ns) ProcessPendingSessionEvents:495(1336ns)}}
2025-11-11T17:21:28.446293Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519200845699702:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.447670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519196550732316:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.482175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699708:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.482190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519200845699678:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(163161851ns) {HandleHandshakeStatus:329(163159726ns) {DropIncomingHandshake:441(14388ns) DropOutgoingHandshake:456(192ns) SwitchToState:178(3761ns) ProcessPendingSessionEvents:495(125574337ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 125403843ns max# 16335896ns}}}
2025-11-11T17:21:28.490244Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519200845699702:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:28.502695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519196550732316:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.502752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:28.502802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519196550732316:2048] [node 1] ICP22 created new session: [5:7571519200845699710:2048]
2025-11-11T17:21:28.502850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519200845699710:2048] [node 1] ICS09 handshake done sender: [5:7571519200845699702:2121] self: [5:7571519200845699707:0] peer: [1:7571519196939575983:0] socket: 49 qp: -1
No RDMA context found for GID: 2025-11-11T17:21:28.510888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519200845699710:2048] [node 1] ICS10 traffic start
::ffff:127.1.0.0
2025-11-11T17:21:28.517859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:28.417102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519194962027804:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.526760Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:28.524250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519196939576013:2108] [node 5] ICS09 handshake done sender: [1:7571519196939575989:2114] self: [1:7571519196939575983:0] peer: [5:7571519200845699707:0] socket: 105 qp: -1
2025-11-11T17:21:28.526336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519196939576013:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:28.526386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:28.522217Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.542573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699709:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.543219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519200845699677:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519200845699698:2119] held: no
2025-11-11T17:21:28.417156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.523023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519195335786430:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519199630753822:2123] held: no
2025-11-11T17:21:28.525822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519199630753822:2123] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:28.417225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519194962027804:2048] [node 1] ICP22 created new session: [3:7571519199256995195:2048]
2025-11-11T17:21:28.529696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519195335786430:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.550741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699712:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.533666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:28.533732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519195335786430:2049] [node 2] ICP22 created new session: [6:7571519199630753828:2049]
2025-11-11T17:21:28.533796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519199630753828:2049] [node 2] ICS09 handshake done sender: [6:7571519199630753822:2123] self: [6:7571519199630753826:0] peer: [2:7571519197284957835:0] socket: 79 qp: -1
2025-11-11T17:21:28.533850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519199630753828:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:28.533897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:28.559288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519192644608683:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(65780794ns) {HandleHandshakeStatus:329(65777124ns) {DropIncomingHandshake:441(13749ns) DropOutgoingHandshake:456(262ns) SwitchToState:178(456ns) ProcessPendingSessionEvents:495(613644ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 557604ns max# 80702ns}}}
2025-11-11T17:21:28.417275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519199256995195:2048] [node 1] ICS09 handshake done sender: [3:7571519199256995178:2116] self: [3:7571519199256995188:0] peer: [1:7571519192644608679:0] socket: 89 qp: -1
2025-11-11T17:21:28.569810Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519200845699712:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.417314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519199256995195:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:28.417351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:28.478259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519194962027868:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(111015735ns) {HandleHandshakeStatus:329(111013035ns) {DropIncomingHandshake:441(12169ns) DropOutgoingHandshake:456(267ns) SwitchToState:178(439ns) ProcessPendingSessionEvents:495(110575111ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 107494803ns max# 18466136ns}}}
2025-11-11T17:21:28.502547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519194962027804:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(81741600ns) {HandleHandshakeStatus:329(81738534ns) {DropIncomingHandshake:441(1000ns) DropOutgoingHandshake:456(7211ns) SwitchToState:178(634ns) ProcessPendingSessionEvents:495(81347963ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 81167362ns max# 9139547ns}}}
2025-11-11T17:21:28.520233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519194962027804:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:28.521513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519199256995197:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22539
2025-11-11T17:21:28.528688Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519199256995197:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.551748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519194962027805:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.565832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519194962027805:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(22981249ns) {IncomingHandshake:261(22979399ns) {DropIncomingHandshake:441(1497ns) PrepareNewSessionHandshake:476(4056ns) {SwitchToState:178(2703ns)} GenerateSessionVirtualId:988(984ns) IssueIncomingHandshakeReply:193(1951314ns) {SwitchToState:178(3597ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.569995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519196550732318:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:28.570171Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519197284957840:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.570206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519197284957840:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.583705Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519199256995183:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:28.574297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519196550732316:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(60302830ns) {HandleHandshakeStatus:329(60299698ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(6625ns) SwitchToState:178(377ns) ProcessPendingSessionEvents:495(36966388ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36872471ns max# 13782450ns}}}
2025-11-11T17:21:28.590111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519200845699714:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14111
2025-11-11T17:21:28.599198Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519200845699712:2124] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:28.600636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519196550732318:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.600697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:28.600735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519196550732318:2050] [node 3] ICP22 created new session: [5:7571519200845699715:2050]
2025-11-11T17:21:28.600773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519200845699715:2050] [node 3] ICS09 handshake done sender: [5:7571519200845699712:2124] self: [5:7571519200845699713:0] peer: [3:7571519199256995181:0] socket: 53 qp: -1
2025-11-11T17:21:28.600805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519200845699715:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:28.600832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:28.583762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519199256995183:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:28.600086Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519197284957840:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:28.600531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519192989990465:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.600602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:28.600655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519192989990465:2049] [node 3] ICP22 created new session: [2:7571519197284957864:2049]
2025-11-11T17:21:28.600704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519197284957864:2049] [node 3] ICS09 handshake done sender: [2:7571519197284957840:2116] self: [2:7571519197284957838:0] peer: [3:7571519199256995199:0] socket: 65 qp: -1
2025-11-11T17:21:28.600738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519197284957864:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:28.600768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:28.589869Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519199256995183:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:28.634362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519200845699677:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(57148895ns) {HandleHandshakeStatus:395(57145421ns) {LogHandshakeFail:472(18774ns) {UpdateErrorStateLog:544(4860ns)} DropOutgoingHandshake:456(381ns) ProcessPendingSessionEvents:495(57063882ns) {unordered DropSessionEvent:512 num# 100 sum# 18987747ns max# 7735693ns}}}
2025-11-11T17:21:28.634514Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519200845699698:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:28.601699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519194962027867:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:28.635703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519200845699677:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.635803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:28.601798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:28.635868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519200845699677:2109] [node 4] ICP22 created new session: [5:7571519200845699717:2109]
2025-11-11T17:21:28.635913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519200845699717:2109] [node 4] ICS09 handshake done sender: [5:7571519200845699698:2119] self: [5:7571519200845699705:0] peer: [4:7571519198993635168:0] socket: 96 qp: -1
2025-11-11T17:21:28.635953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519200845699717:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:28.636022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:28.601847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519194962027867:2108] [node 5] ICP22 created new session: [3:7571519199256995201:2108]
2025-11-11T17:21:28.645737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519192989990465:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(16328345ns) {HandleHandshakeStatus:329(16325639ns) {DropIncomingHandshake:441(17782ns) DropOutgoingHandshake:456(465ns) SwitchToState:178(525ns) ProcessPendingSessionEvents:495(15929172ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15842769ns max# 9375987ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:28.601893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519199256995201:2108] [node 5] ICS09 handshake done sender: [3:7571519199256995183:2118] self: [3:7571519199256995181:0] peer: [5:7571519200845699713:0] socket: 91 qp: -1
2025-11-11T17:21:28.601923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519199256995201:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:28.601952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:28.629865Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:28.641905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519194962027805:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519199256995197:2120] held: no
2025-11-11T17:21:28.646820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519199256995203:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22539
2025-11-11T17:21:28.651489Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519199256995197:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:28.653851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519194962027805:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:28.653905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:28.653952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519194962027805:2049] [node 2] ICP22 created new session: [3:7571519199256995204:2049]
2025-11-11T17:21:28.654006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519199256995204:2049] [node 2] ICS09 handshake done sender: [3:7571519199256995197:2120] self: [3:7571519199256995199:0] peer: [2:7571519197284957838:0] socket: 90 qp: -1
2025-11-11T17:21:28.654043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519199256995204:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:28.654118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:28.713338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:28.713629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519197284957847:2048] [node 1] ICS01 socket: 59 reason# EndOfStream
2025-11-11T17:21:28.713679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519192989990464:2048] [node 1] ICP30 unregister session Session# [2:7571519197284957847:2048] VirtualId# [2:7571519197284957842:0]
2025-11-11T17:21:28.713715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519197284957847:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.715608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:28.715846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519199256995195:2048] [node 1] ICS07 socket disconnect 89 reason# EndOfStream
2025-11-11T17:21:28.715885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519199256995195:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.715951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:28.715986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519199256995195:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:28.713808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:28.713852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:28.722607Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:21:28.723007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519194962027804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:28.724188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519199256995195:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:28.724711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:28.725327Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address
2025-11-11T17:21:28.725626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519194962027804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:28.726497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519198993635182:2048] [node 1] ICS01 socket: 56 reason# EndOfStream
2025-11-11T17:21:28.726538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519194698667797:2048] [node 1] ICP30 unregister session Session# [4:7571519198993635182:2048] VirtualId# [4:7571519198993635172:0]
2025-11-11T17:21:28.726569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519198993635182:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.726637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:28.726677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:28.725674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519199256995195:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:28.725703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519194962027804:2048] [node 1] ICP30 unregister session Session# [3:7571519199256995195:2048] VirtualId# [3:7571519199256995188:0]
2025-11-11T17:21:28.725797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:28.725843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519194962027804:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.725861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.733314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:28.733726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:28.733931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519199630753824:2048] [node 1] ICS07 socket disconnect 61 reason# EndOfStream
2025-11-11T17:21:28.733960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519199630753824:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.734036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:28.734076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519199630753824:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:28.736867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519194962027804:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:28.733629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519200845699710:2048] [node 1] ICS07 socket disconnect 49 reason# EndOfStream
2025-11-11T17:21:28.733669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519200845699710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.733765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:28.733803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519200845699710:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:28.734623Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address
2025-11-11T17:21:28.734893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519196550732316:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:28.741069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519200845699710:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:28.741126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519196550732316:2048] [node 1] ICP30 unregister session Session# [5:7571519200845699710:2048] VirtualId# [5:7571519200845699707:0]
2025-11-11T17:21:28.740313Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address
2025-11-11T17:21:28.741183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519195335786429:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:28.741578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519199630753824:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:28.741625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519195335786429:2048] [node 1] ICP30 unregister session Session# [6:7571519199630753824:2048] VirtualId# [6:7571519199630753809:0]
2025-11-11T17:21:28.741756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:28.741792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519195335786429:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.741807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.742493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:28.742542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519196550732316:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.742554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.749197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519195335786429:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:28.753746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519196550732316:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:28.764663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:28.764911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519199256995204:2049] [node 2] ICS01 socket: 90 reason# EndOfStream
2025-11-11T17:21:28.764967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519194962027805:2049] [node 2] ICP30 unregister session Session# [3:7571519199256995204:2049] VirtualId# [3:7571519199256995199:0]
2025-11-11T17:21:28.765333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519199256995204:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.765440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:28.765490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:28.768933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:28.769149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519198993635191:2049] [node 2] ICS01 socket: 68 reason# EndOfStream
2025-11-11T17:21:28.769187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519194698667798:2049] [node 2] ICP30 unregister session Session# [4:7571519198993635191:2049] VirtualId# [4:7571519198993635184:0]
2025-11-11T17:21:28.769228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519198993635191:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.769688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:28.769326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:28.769365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:28.769859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519200845699704:2049] [node 2] ICS01 socket: 72 reason# EndOfStream
2025-11-11T17:21:28.769900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519196550732317:2049] [node 2] ICP30 unregister session Session# [5:7571519200845699704:2049] VirtualId# [5:7571519200845699693:0]
2025-11-11T17:21:28.769935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519200845699704:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.769994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:28.770032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:28.770370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:28.770563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519199630753828:2049] [node 2] ICS01 socket: 79 reason# EndOfStream
2025-11-11T17:21:28.770601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519195335786430:2049] [node 2] ICP30 unregister session Session# [6:7571519199630753828:2049] VirtualId# [6:7571519199630753826:0]
2025-11-11T17:21:28.770639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519199630753828:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.770715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:28.770757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:28.820355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:28.820562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519198993635188:2050] [node 3] ICS01 socket: 82 reason# EndOfStream
2025-11-11T17:21:28.820611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519194698667799:2050] [node 3] ICP30 unregister session Session# [4:7571519198993635188:2050] VirtualId# [4:7571519198993635185:0]
2025-11-11T17:21:28.820645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519198993635188:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.820737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:28.820781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:28.821481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:28.821528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:28.821708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519200845699715:2050] [node 3] ICS07 socket disconnect 53 reason# EndOfStream
2025-11-11T17:21:28.821743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519200845699715:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.821802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519199630753820:2050] [node 3] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-11T17:21:28.821847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519199630753820:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.821921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:28.821814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:28.821877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519200845699715:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:28.822624Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address
2025-11-11T17:21:28.822927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519199630753820:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:28.822898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519196550732318:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:28.823036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519200845699715:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:28.823146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519196550732318:2050] [node 3] ICP30 unregister session Session# [5:7571519200845699715:2050] VirtualId# [5:7571519200845699713:0]
2025-11-11T17:21:28.823243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:28.823282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519196550732318:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.823297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.832734Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address
2025-11-11T17:21:28.832854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519195335786431:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:28.832904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519199630753820:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:28.838336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519195335786431:2050] [node 3] ICP30 unregister session Session# [6:7571519199630753820:2050] VirtualId# [6:7571519199630753814:0]
2025-11-11T17:21:28.838406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:28.838464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519195335786431:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.838481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:28.833647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519196550732318:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:28.851055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519195335786431:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:28.878742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:28.879010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519200845699717:2109] [node 4] ICS01 socket: 96 reason# EndOfStream
2025-11-11T17:21:28.879059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519200845699677:2109] [node 4] ICP30 unregister session Session# [5:7571519200845699717:2109] VirtualId# [5:7571519200845699705:0]
2025-11-11T17:21:28.879095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519200845699717:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.885735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:28.885977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519199630753808:2099] [node 4] ICS01 socket: 58 reason# EndOfStream
2025-11-11T17:21:28.886023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519195335786486:2099] [node 4] ICP30 unregister session Session# [6:7571519199630753808:2099] VirtualId# [6:7571519199630753805:0]
2025-11-11T17:21:28.886060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519199630753808:2099] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.886161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:28.886213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:28.879178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:28.879230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:28.965766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:28.966001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519199630753817:2101] [node 5] ICS01 socket: 71 reason# EndOfStream
2025-11-11T17:21:28.966050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519195335786496:2101] [node 5] ICP30 unregister session Session# [6:7571519199630753817:2101] VirtualId# [6:7571519199630753813:0]
2025-11-11T17:21:28.966093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519199630753817:2101] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:28.966164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:28.966205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:29.058425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519204638141593:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:29.058470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519204638141594:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:29.079989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519202187527956:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:29.080027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519202187527957:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:29.107137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519201292332933:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:29.107183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519201292332934:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:29.137315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519204853647193:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:29.137352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519204853647195:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:29.137513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519204853647194:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:29.161037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519204868669571:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:29.161062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519204868669572:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:29.161443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519204868669570:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:29.181534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519203717309061:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:29.182995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519203717309059:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:29.183100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519203717309060:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:29.345167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519204638141659:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:29.332623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519203717309113:2099] [node 5] ICP01 ready to work
2025-11-11T17:21:29.377739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519204638141594:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21046567ns) {Configure:104(21039732ns) {ProcessConfigured:135(79594ns) {StartInitialHandshake:157(75958ns) {DropHandshakes:469(3230ns) {DropIncomingHandshake:441(720ns) DropOutgoingHandshake:456(672ns)} GenerateSessionVirtualId:988(1321ns) PrepareNewSessionHandshake:476(2164ns) {SwitchToState:178(641ns)}}}}}
2025-11-11T17:21:29.369866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519203717309118:2102] [node 4] ICP01 ready to work
2025-11-11T17:21:29.405778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519202187528019:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:29.406136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519201292332998:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:29.437656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519204853647256:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:29.391248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519204638141661:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:29.446872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519202187528020:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:29.461996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519201292332999:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:29.428173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519204638141660:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:29.454447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519204853647257:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:29.462659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519204868669634:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:29.453704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519202187528021:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:29.465994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519202187528020:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16599454ns) {Configure:104(16592767ns) {ProcessConfigured:135(39703ns) {StartInitialHandshake:157(37155ns) {DropHandshakes:469(2741ns) {DropIncomingHandshake:441(706ns) DropOutgoingHandshake:456(782ns)} GenerateSessionVirtualId:988(473ns) PrepareNewSessionHandshake:476(825ns) {SwitchToState:178(307ns)}}}}}
2025-11-11T17:21:29.463888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519201292333000:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:29.464228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519204868669635:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:29.474806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519204638141593:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(112983064ns) {Configure:104(112976155ns) {ProcessConfigured:135(6151620ns) {StartInitialHandshake:157(6149358ns) {DropHandshakes:469(2847ns) {DropIncomingHandshake:441(552ns) DropOutgoingHandshake:456(657ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:476(3377ns) {SwitchToState:178(1808ns)}}}}}
2025-11-11T17:21:29.473964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519204853647194:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(107896371ns) {Configure:104(107886418ns) {ProcessConfigured:135(5168143ns) {StartInitialHandshake:157(5164614ns) {DropHandshakes:469(1841ns) {DropIncomingHandshake:441(390ns) DropOutgoingHandshake:456(285ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:476(5052ns) {SwitchToState:178(2290ns)}}}}}
2025-11-11T17:21:29.485472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519202187528019:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(22228167ns) {Configure:104(22223597ns) {ProcessConfigured:135(47133ns) {StartInitialHandshake:157(44715ns) {DropHandshakes:469(2828ns) {DropIncomingHandshake:441(614ns) DropOutgoingHandshake:456(371ns)} GenerateSessionVirtualId:988(584ns) PrepareNewSessionHandshake:476(1522ns) {SwitchToState:178(681ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.487407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519202187528030:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40002
2025-11-11T17:21:29.488988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519203717309118:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26253433ns) {Configure:104(26246936ns) {ProcessConfigured:135(3334771ns) {StartInitialHandshake:157(3333101ns) {DropHandshakes:469(3371ns) {DropIncomingHandshake:441(727ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(1070ns) PrepareNewSessionHandshake:476(4222ns) {SwitchToState:178(2107ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.511007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519204868669572:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20710885ns) {Configure:104(20706238ns) {ProcessConfigured:135(55213ns) {StartInitialHandshake:157(52503ns) {DropHandshakes:469(2574ns) {DropIncomingHandshake:441(431ns) DropOutgoingHandshake:456(433ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:476(1306ns) {SwitchToState:178(575ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.514366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519202187528033:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40002
2025-11-11T17:21:29.499415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519204638141665:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26673
2025-11-11T17:21:29.506046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647266:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.529868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519202187528032:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40002
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.507577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519204853647195:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(45592623ns) {Configure:104(45585772ns) {ProcessConfigured:135(6140815ns) {StartInitialHandshake:157(6137521ns) {DropHandshakes:469(3403ns) {DropIncomingHandshake:441(616ns) DropOutgoingHandshake:456(604ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:476(3225ns) {SwitchToState:178(1236ns)}}}}}
2025-11-11T17:21:29.537903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519201292332998:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(73836716ns) {Configure:104(73290542ns) {ProcessConfigured:135(16986362ns) {StartInitialHandshake:157(16983210ns) {DropHandshakes:469(2422ns) {DropIncomingHandshake:441(637ns) DropOutgoingHandshake:456(584ns)} GenerateSessionVirtualId:988(649ns) PrepareNewSessionHandshake:476(8485ns) {SwitchToState:178(2346ns)}}}}}
2025-11-11T17:21:29.546680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519201292332933:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(68300196ns) {Configure:104(68294715ns) {ProcessConfigured:135(9380303ns) {StartInitialHandshake:157(9377458ns) {DropHandshakes:469(1186ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(253ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:476(3545ns) {SwitchToState:178(1769ns)}}}}}
2025-11-11T17:21:29.530307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519204868669635:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(56225317ns) {Configure:104(56220471ns) {ProcessConfigured:135(7581982ns) {StartInitialHandshake:157(7578727ns) {DropHandshakes:469(2542ns) {DropIncomingHandshake:441(882ns) DropOutgoingHandshake:456(515ns)} GenerateSessionVirtualId:988(983ns) PrepareNewSessionHandshake:476(4307ns) {SwitchToState:178(1690ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.564267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519201292333004:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22630
2025-11-11T17:21:29.549732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528032:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.542314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519203717309060:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(79057155ns) {Configure:104(79052440ns) {ProcessConfigured:135(15712217ns) {StartInitialHandshake:157(15708968ns) {DropHandshakes:469(2785ns) {DropIncomingHandshake:441(654ns) DropOutgoingHandshake:456(539ns)} GenerateSessionVirtualId:988(874ns) PrepareNewSessionHandshake:476(3429ns) {SwitchToState:178(1441ns)}}}}}
2025-11-11T17:21:29.589965Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333004:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.591694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519201292332933:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.531321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519204853647256:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32897863ns) {Configure:104(32892444ns) {ProcessConfigured:135(7318420ns) {StartInitialHandshake:157(7315652ns) {DropHandshakes:469(2694ns) {DropIncomingHandshake:441(412ns) DropOutgoingHandshake:456(494ns)} GenerateSessionVirtualId:988(714ns) PrepareNewSessionHandshake:476(2958ns) {SwitchToState:178(729ns)}}}}}
2025-11-11T17:21:29.593650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519202187527956:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(71601448ns) {Configure:104(71597372ns) {ProcessConfigured:135(14336469ns) {StartInitialHandshake:157(14334285ns) {DropHandshakes:469(2161ns) {DropIncomingHandshake:441(863ns) DropOutgoingHandshake:456(350ns)} GenerateSessionVirtualId:988(606ns) PrepareNewSessionHandshake:476(3016ns) {SwitchToState:178(473ns)}}}}}
2025-11-11T17:21:29.579314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519204638141659:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(103992981ns) {Configure:104(103988604ns) {ProcessConfigured:135(10598217ns) {StartInitialHandshake:157(10595478ns) {DropHandshakes:469(2890ns) {DropIncomingHandshake:441(501ns) DropOutgoingHandshake:456(787ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:476(4268ns) {SwitchToState:178(1673ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.605216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519201292333016:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22630
2025-11-11T17:21:29.597647Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519204638141665:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.553058Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519204853647266:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.545001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309131:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:29.595694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519204868669570:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(95536280ns) {Configure:104(95530984ns) {ProcessConfigured:135(32217149ns) {StartInitialHandshake:157(32214897ns) {DropHandshakes:469(606ns) {DropIncomingHandshake:441(44ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(3700ns) {SwitchToState:178(834ns)}}}}}
2025-11-11T17:21:29.605469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519201292333002:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22630
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.625753Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333016:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.566984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519204853647257:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(91683746ns) {Configure:104(91677353ns) {ProcessConfigured:135(15940703ns) {StartInitialHandshake:157(15937896ns) {DropHandshakes:469(2311ns) {DropIncomingHandshake:441(651ns) DropOutgoingHandshake:456(413ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:476(4148ns) {SwitchToState:178(1746ns)}}}}}
2025-11-11T17:21:29.622047Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519204638141657:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.625949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528025:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.626000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519202187528025:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.642838Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528029:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.642863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519202187528029:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.650090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519202187528021:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(168869168ns) {Configure:104(168860747ns) {ProcessConfigured:135(6123592ns) {StartInitialHandshake:157(6120416ns) {DropHandshakes:469(2467ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(729ns) PrepareNewSessionHandshake:476(7522ns) {SwitchToState:178(1099ns)}}}}}
2025-11-11T17:21:29.650248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519202187527956:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.600221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519204868669634:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(76201114ns) {Configure:104(76196871ns) {ProcessConfigured:135(22984396ns) {StartInitialHandshake:157(22981660ns) {DropHandshakes:469(1529ns) {DropIncomingHandshake:441(625ns) DropOutgoingHandshake:456(234ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:476(5751ns) {SwitchToState:178(2453ns)}}}}}
2025-11-11T17:21:29.545995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519203717309113:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(83554496ns) {Configure:104(83548482ns) {ProcessConfigured:135(13953302ns) {StartInitialHandshake:157(13952206ns) {DropHandshakes:469(966ns) {DropIncomingHandshake:441(267ns) DropOutgoingHandshake:456(183ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1613ns) {SwitchToState:178(97ns)}}}}}
2025-11-11T17:21:29.581306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519204853647194:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.661994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519202187528025:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:29.662626Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519202187528029:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:29.662913Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528030:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.663074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519202187528020:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.663118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:29.663163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519202187528020:2107] [node 4] ICP22 created new session: [2:7571519202187528043:2107]
2025-11-11T17:21:29.663211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519202187528043:2107] [node 4] ICS09 handshake done sender: [2:7571519202187528025:2111] self: [2:7571519202187528024:0] peer: [4:7571519204853647274:0] socket: 55 qp: -1
2025-11-11T17:21:29.622110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519204638141657:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.646941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519201292333017:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22630
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.549732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519203717309059:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(120142406ns) {Configure:104(120134352ns) {ProcessConfigured:135(16182659ns) {StartInitialHandshake:157(16180572ns) {DropHandshakes:469(2379ns) {DropIncomingHandshake:441(583ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:476(3180ns) {SwitchToState:178(1595ns)}}}}}
2025-11-11T17:21:29.625141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519204638141670:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26673
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.590940Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519203717309131:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.581578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647272:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.672005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519201292333018:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22630
2025-11-11T17:21:29.677925Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528033:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.597683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519203717309060:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.707653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309142:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:29.650876Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519204638141663:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.672473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333017:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.672747Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333002:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.697656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519201292332934:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.650901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519204638141663:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.651735Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519204638141663:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:29.652126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519204638141593:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.652177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:29.652226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519204638141593:2048] [node 2] ICP22 created new session: [1:7571519204638141676:2048]
2025-11-11T17:21:29.652271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519204638141676:2048] [node 2] ICS09 handshake done sender: [1:7571519204638141663:2112] self: [1:7571519204638141662:0] peer: [2:7571519202187528041:0] socket: 64 qp: -1
2025-11-11T17:21:29.652305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519204638141676:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:29.652342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:29.654176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519204638141657:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:29.662197Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519204638141670:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.723682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309140:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:29.581656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647273:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
2025-11-11T17:21:29.717984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519204868669571:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(206211310ns) {Configure:104(206205874ns) {ProcessConfigured:135(19936756ns) {StartInitialHandshake:157(19933601ns) {DropHandshakes:469(1716544ns) {DropIncomingHandshake:441(2534ns) DropOutgoingHandshake:456(1004ns)} GenerateSessionVirtualId:988(1950ns) PrepareNewSessionHandshake:476(3371ns) {SwitchToState:178(1486ns)}}}}}
2025-11-11T17:21:29.723729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519204868669647:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:29.678123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519202187528043:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:29.685689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519204638141594:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.682365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519202187528019:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.727559Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519204868669647:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.690038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:29.694322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:29.698177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519202187528042:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40002
2025-11-11T17:21:29.717914Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528023:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.717950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519202187528023:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.722486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519202187528036:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40002
2025-11-11T17:21:29.724001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519202187528019:2109] [node 6] ICP22 created new session: [2:7571519202187528044:2109]
2025-11-11T17:21:29.724069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519202187528044:2109] [node 6] ICS09 handshake done sender: [2:7571519202187528029:2112] self: [2:7571519202187528028:0] peer: [6:7571519203717309138:0] socket: 62 qp: -1
2025-11-11T17:21:29.724133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519202187528044:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:29.724208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:29.730010Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528036:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.730220Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519202187528023:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:29.733666Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519202187528032:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:29.616982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519204853647273:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.723857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519201292333012:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22630
2025-11-11T17:21:29.685818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:29.685876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519204638141594:2049] [node 3] ICP22 created new session: [1:7571519204638141680:2049]
2025-11-11T17:21:29.685924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519204638141680:2049] [node 3] ICS09 handshake done sender: [1:7571519204638141657:2110] self: [1:7571519204638141656:0] peer: [3:7571519201292333015:0] socket: 61 qp: -1
2025-11-11T17:21:29.685967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519204638141680:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:29.686007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:29.702575Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519204638141660:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(181764323ns) {Configure:104(181757147ns) {ProcessConfigured:135(82398ns) {StartInitialHandshake:157(77621ns) {DropHandshakes:469(2534ns) {DropIncomingHandshake:441(522ns) DropOutgoingHandshake:456(424ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:476(1646ns) {SwitchToState:178(493ns)}}}}}
2025-11-11T17:21:29.733636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333012:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.693923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647276:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
2025-11-11T17:21:29.746454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519202187527956:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.757574Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519204638141668:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.738214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519204868669570:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.750992Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519203717309140:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.713702Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519204853647272:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.737840Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519201292333004:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:29.746534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.757651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519204638141668:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.757420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309139:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:29.781772Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519204868669644:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.724182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
2025-11-11T17:21:29.761808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519203717309113:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.746583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519202187527956:2048] [node 1] ICP22 created new session: [2:7571519202187528051:2048]
2025-11-11T17:21:29.769438Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519204638141668:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:29.742031Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:29.750872Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519204853647277:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.758953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519204853647193:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.759532Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519204853647266:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:29.767534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647280:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
2025-11-11T17:21:29.785081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519204853647194:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.814230Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519204853647280:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.814379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519204853647195:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.746629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519202187528051:2048] [node 1] ICS09 handshake done sender: [2:7571519202187528032:2114] self: [2:7571519202187528041:0] peer: [1:7571519204638141662:0] socket: 66 qp: -1
2025-11-11T17:21:29.781855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519204868669644:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.746668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519202187528051:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:29.746702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:29.781752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519202187527957:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.789912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:29.797936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519202187527957:2049] [node 3] ICP22 created new session: [2:7571519202187528053:2049]
2025-11-11T17:21:29.805904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519202187528053:2049] [node 3] ICS09 handshake done sender: [2:7571519202187528023:2110] self: [2:7571519202187528022:0] peer: [3:7571519201292333021:0] socket: 54 qp: -1
2025-11-11T17:21:29.817447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519202187528053:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:29.817517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.774335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519204638141659:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.777733Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519203717309139:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.826076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:29.749769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519201292332933:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519201292333004:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.781804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:29.799024Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519204868669644:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:29.761672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519201292332933:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.762406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:29.767129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519201292333024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22630
2025-11-11T17:21:29.770279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519201292332933:2048] [node 1] ICP22 created new session: [3:7571519201292333025:2048]
2025-11-11T17:21:29.781885Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519201292333002:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:29.801723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519201292333025:2048] [node 1] ICS09 handshake done sender: [3:7571519201292333004:2112] self: [3:7571519201292333015:0] peer: [1:7571519204638141656:0] socket: 65 qp: -1
2025-11-11T17:21:29.882202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519204868669634:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.882348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.833951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519204853647194:2049] [node 2] ICP22 created new session: [4:7571519204853647281:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.885745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519204868669651:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:29.793381Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519203717309142:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.802728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333009:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.846059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647284:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
2025-11-11T17:21:29.898259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519204868669634:2109] [node 6] ICP22 created new session: [5:7571519204868669654:2109]
2025-11-11T17:21:29.797658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519203717309061:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-11T17:21:29.789877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519204638141659:2108] [node 5] ICP22 created new session: [1:7571519204638141687:2108]
::ffff:127.1.0.0
2025-11-11T17:21:29.798618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309145:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:29.805934Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519204638141672:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.802781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519201292333009:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.857908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519204853647281:2049] [node 2] ICS09 handshake done sender: [4:7571519204853647266:2113] self: [4:7571519204853647274:0] peer: [2:7571519202187528024:0] socket: 63 qp: -1
2025-11-11T17:21:29.964141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519202187527957:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(189227627ns) {HandleHandshakeStatus:329(189224578ns) {DropIncomingHandshake:441(25420ns) DropOutgoingHandshake:456(450ns) SwitchToState:178(575ns) ProcessPendingSessionEvents:495(105417560ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 105312682ns max# 36893505ns}}}
2025-11-11T17:21:29.906493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519204868669652:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:29.802797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519201292332934:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.801639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309141:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:29.946620Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519204868669647:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:29.878052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519204853647281:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.805978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519204638141672:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.802855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:29.806174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519204638141687:2108] [node 5] ICS09 handshake done sender: [1:7571519204638141668:2114] self: [1:7571519204638141667:0] peer: [5:7571519204868669650:0] socket: 79 qp: -1
2025-11-11T17:21:29.809930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519203717309061:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(19753639ns) {IncomingHandshake:261(19751130ns) {DropIncomingHandshake:441(630ns) PrepareNewSessionHandshake:476(1628ns) {SwitchToState:178(669ns)} GenerateSessionVirtualId:988(711ns) IssueIncomingHandshakeReply:193(7427062ns) {SwitchToState:178(4142ns)}}}
2025-11-11T17:21:29.955659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519204868669654:2109] [node 6] ICS09 handshake done sender: [5:7571519204868669644:2113] self: [5:7571519204868669639:0] peer: [6:7571519203717309143:0] socket: 90 qp: -1
2025-11-11T17:21:29.891506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519204853647286:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58226
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:30.042495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519202187527956:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(139293300ns) {HandleHandshakeStatus:329(139284097ns) {DropIncomingHandshake:441(624ns) DropOutgoingHandshake:456(5297ns) SwitchToState:178(538ns) ProcessPendingSessionEvents:495(138826961ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 130811607ns max# 23603352ns}}}
2025-11-11T17:21:30.044100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519206482495351:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40002
2025-11-11T17:21:29.810311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519204638141687:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:29.802902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519201292332934:2049] [node 2] ICP22 created new session: [3:7571519201292333026:2049]
2025-11-11T17:21:29.894029Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519204853647277:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:29.815126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519203717309059:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:29.955718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519204868669654:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:30.057666Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519202187528038:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.057737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519202187528038:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.829699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:29.894266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:29.970182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:29.806339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519201292333025:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.873922Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519203717309131:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:29.833788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519204638141685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26673
2025-11-11T17:21:29.910171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519204853647280:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:29.918373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519204853647193:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.940216Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.941191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519204853647195:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.941274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:29.941323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519204853647195:2050] [node 3] ICP22 created new session: [4:7571519204853647287:2050]
2025-11-11T17:21:29.941366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519204853647287:2050] [node 3] ICS09 handshake done sender: [4:7571519204853647280:2120] self: [4:7571519204853647282:0] peer: [3:7571519201292333010:0] socket: 107 qp: -1
2025-11-11T17:21:29.941405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519204853647287:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:29.941437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:29.813751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519201292333026:2049] [node 2] ICS09 handshake done sender: [3:7571519201292333002:2111] self: [3:7571519201292333021:0] peer: [2:7571519202187528022:0] socket: 57 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.973974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519204868669570:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.839832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519204638141683:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.874435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309147:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:30.093443Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519202187528038:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:29.839894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519204638141683:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.846356Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 2] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:29.848005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519204638141672:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:29.855251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519204638141593:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:29.885625Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519204638141683:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:29.888945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519204638141659:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(118098929ns) {HandleHandshakeStatus:329(118095162ns) {DropIncomingHandshake:441(1302ns) DropOutgoingHandshake:456(1189ns) SwitchToState:178(1035ns) ProcessPendingSessionEvents:495(6886267ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6811881ns max# 6277176ns}}}
2025-11-11T17:21:29.898658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519204638141661:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.910957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:29.911020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519204638141661:2107] [node 4] ICP22 created new session: [1:7571519204638141691:2107]
2025-11-11T17:21:29.950090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519204638141660:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.950185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:29.955329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519204638141660:2109] [node 6] ICP22 created new session: [1:7571519204638141692:2109]
2025-11-11T17:21:29.969820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519204638141691:2107] [node 4] ICS09 handshake done sender: [1:7571519204638141672:2116] self: [1:7571519204638141671:0] peer: [4:7571519204853647279:0] socket: 94 qp: -1
2025-11-11T17:21:29.969961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519204638141692:2109] [node 6] ICS09 handshake done sender: [1:7571519204638141683:2117] self: [1:7571519204638141682:0] peer: [6:7571519203717309146:0] socket: 84 qp: -1
2025-11-11T17:21:29.975132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519204638141691:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:29.975186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:29.986382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519204638141692:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:29.992616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:30.010435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519204638141694:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26673
2025-11-11T17:21:30.023932Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:30.109258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519208933108992:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26673
2025-11-11T17:21:30.116633Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:30.123176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519204638141661:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(227467944ns) {HandleHandshakeStatus:329(227465131ns) {DropIncomingHandshake:441(17903ns) DropOutgoingHandshake:456(342ns) SwitchToState:178(991ns) ProcessPendingSessionEvents:495(108448315ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 108335036ns max# 23361564ns}}}
2025-11-11T17:21:29.975087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:29.815224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519201292333026:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:29.985069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519204853647193:2048] [node 1] ICP22 created new session: [4:7571519204853647289:2048]
2025-11-11T17:21:29.991484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519204853647289:2048] [node 1] ICS09 handshake done sender: [4:7571519204853647277:2119] self: [4:7571519204853647279:0] peer: [1:7571519204638141671:0] socket: 99 qp: -1
2025-11-11T17:21:30.003252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519204853647289:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:30.013817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:30.031844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519204853647193:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(124189080ns) {HandleHandshakeStatus:329(124184990ns) {DropIncomingHandshake:441(2281ns) DropOutgoingHandshake:456(5201ns) SwitchToState:178(4345ns) ProcessPendingSessionEvents:495(8975937ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8905355ns max# 6611940ns}}}
2025-11-11T17:21:30.031932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519204853647193:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:30.069795Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519204853647268:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.069854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519204853647268:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:30.077887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519204853647271:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.077917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519204853647271:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:30.082055Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519204853647268:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:30.083998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519204853647194:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(199259857ns) {HandleHandshakeStatus:329(199256318ns) {DropIncomingHandshake:441(829ns) DropOutgoingHandshake:456(15373ns) SwitchToState:178(535ns) ProcessPendingSessionEvents:495(28740441ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28684585ns max# 28270630ns}}}
2025-11-11T17:21:30.110485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519204853647256:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.870275Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519203717309140:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:30.106553Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519206482495351:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.974106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:29.821773Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519201292333009:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:30.156163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:30.163522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519204853647195:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(221242489ns) {HandleHandshakeStatus:329(221239597ns) {DropIncomingHandshake:441(1069ns) DropOutgoingHandshake:456(4000ns) SwitchToState:178(656ns) ProcessPendingSessionEvents:495(220855051ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 215885260ns max# 17729286ns}}}
2025-11-11T17:21:30.170062Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519204853647271:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:30.171937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519204853647257:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:30.177824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519204853647256:2109] [node 6] ICP22 created new session: [4:7571519209148614590:2109]
2025-11-11T17:21:29.929947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309149:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:29.974173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519204868669570:2048] [node 1] ICP22 created new session: [5:7571519204868669658:2048]
2025-11-11T17:21:30.118504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519202187528021:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:30.192119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519209148614590:2109] [node 6] ICS09 handshake done sender: [4:7571519204853647268:2114] self: [4:7571519204853647265:0] peer: [6:7571519208012276455:0] socket: 113 qp: -1
2025-11-11T17:21:30.206003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519209148614590:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:30.127078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:29.826125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:29.931774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519203717309060:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.974218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519204868669658:2048] [node 1] ICS09 handshake done sender: [5:7571519204868669647:2115] self: [5:7571519204868669650:0] peer: [1:7571519204638141667:0] socket: 87 qp: -1
2025-11-11T17:21:30.222018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:30.237744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:30.237822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519204853647257:2108] [node 5] ICP22 created new session: [4:7571519209148614591:2108]
2025-11-11T17:21:30.227460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519204638141660:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(315699737ns) {HandleHandshakeStatus:329(315697184ns) {DropIncomingHandshake:441(3454ns) DropOutgoingHandshake:456(2019ns) SwitchToState:178(779ns) ProcessPendingSessionEvents:495(209597176ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 209429332ns max# 17175843ns}}}
2025-11-11T17:21:29.974256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519204868669658:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:29.829732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:30.140910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519202187528021:2108] [node 5] ICP22 created new session: [2:7571519206482495354:2108]
2025-11-11T17:21:29.931829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:30.247489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519209148614591:2108] [node 5] ICS09 handshake done sender: [4:7571519204853647271:2115] self: [4:7571519204853647270:0] peer: [5:7571519209163636960:0] socket: 76 qp: -1
2025-11-11T17:21:30.146553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519206482495354:2108] [node 5] ICS09 handshake done sender: [2:7571519202187528038:2118] self: [2:7571519202187528037:0] peer: [5:7571519209163636958:0] socket: 101 qp: -1
2025-11-11T17:21:29.851670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519201292332934:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(47919535ns) {HandleHandshakeStatus:329(47912532ns) {DropIncomingHandshake:441(986ns) DropOutgoingHandshake:456(5284ns) SwitchToState:178(505ns) ProcessPendingSessionEvents:495(553939ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 498467ns max# 62836ns}}}
2025-11-11T17:21:29.974298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:30.228541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519204638141660:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:30.253243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519209148614591:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:30.258257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:29.931878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519203717309060:2049] [node 2] ICP22 created new session: [6:7571519203717309150:2049]
2025-11-11T17:21:30.275640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519204853647257:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(103472088ns) {HandleHandshakeStatus:329(103467838ns) {DropIncomingHandshake:441(17168ns) DropOutgoingHandshake:456(586ns) SwitchToState:178(660ns) ProcessPendingSessionEvents:495(2347514ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 2284024ns max# 1670566ns}}}
2025-11-11T17:21:29.851974Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333011:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:29.931924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519203717309150:2049] [node 2] ICS09 handshake done sender: [6:7571519203717309131:2113] self: [6:7571519203717309138:0] peer: [2:7571519202187528028:0] socket: 72 qp: -1
2025-11-11T17:21:30.169794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519206482495354:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:30.014285Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519204868669651:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.014497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519204868669657:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:30.014961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519204868669571:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:30.015040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519209163636956:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:30.037627Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519204868669657:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.037791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519204868669572:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:30.069952Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519209163636956:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.073960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519204868669635:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:30.086198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519209163636961:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:30.098210Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519204868669651:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:30.098437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519204868669571:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:30.098527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:30.098583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519204868669571:2049] [node 2] ICP22 created new session: [5:7571519209163636962:2049]
2025-11-11T17:21:30.098634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519209163636962:2049] [node 2] ICS09 handshake done sender: [5:7571519204868669651:2117] self: [5:7571519209163636958:0] peer: [2:7571519202187528037:0] socket: 105 qp: -1
2025-11-11T17:21:30.098676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519209163636962:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:30.098739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:30.126054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519209163636964:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:30.151540Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519204868669657:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:30.187108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519204868669572:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:30.187246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:30.187309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519204868669572:2050] [node 3] ICP22 created new session: [5:7571519209163636966:2050]
2025-11-11T17:21:30.187399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519209163636966:2050] [node 3] ICS09 handshake done sender: [5:7571519204868669657:2119] self: [5:7571519209163636959:0] peer: [3:7571519201292333005:0] socket: 60 qp: -1
2025-11-11T17:21:30.187444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519209163636966:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:30.187479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:30.188980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519209163636965:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48946
2025-11-11T17:21:30.190913Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519209163636956:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:30.191076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519204868669635:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:30.191141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:30.191189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519204868669635:2108] [node 4] ICP22 created new session: [5:7571519209163636968:2108]
2025-11-11T17:21:30.191256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519209163636968:2108] [node 4] ICS09 handshake done sender: [5:7571519209163636956:2120] self: [5:7571519209163636960:0] peer: [4:7571519204853647270:0] socket: 83 qp: -1
2025-11-11T17:21:30.191295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519209163636968:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:30.191319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:30.207705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519204868669634:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(324501720ns) {HandleHandshakeStatus:329(324496529ns) {DropIncomingHandshake:441(736ns) DropOutgoingHandshake:456(948ns) SwitchToState:178(884ns) ProcessPendingSessionEvents:495(201159931ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 200987910ns max# 21048592ns}}}
2025-11-11T17:21:30.288393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519204868669635:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(84461095ns) {HandleHandshakeStatus:329(84458377ns) {DropIncomingHandshake:441(410ns) DropOutgoingHandshake:456(2847ns) SwitchToState:178(625ns) ProcessPendingSessionEvents:495(84069549ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 79345387ns max# 16481593ns}}}
2025-11-11T17:21:30.288518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519204868669570:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(80273546ns) {HandleHandshakeStatus:329(80269008ns) {DropIncomingHandshake:441(921ns) DropOutgoingHandshake:456(6871ns) SwitchToState:178(587ns) ProcessPendingSessionEvents:495(79779330ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 79670332ns max# 16605267ns}}}
2025-11-11T17:21:29.851997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519201292333011:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:29.885753Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519201292333011:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:29.892527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519201292333000:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.892630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:29.892689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519201292333000:2110] [node 6] ICP22 created new session: [3:7571519201292333031:2110]
2025-11-11T17:21:29.892735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519201292333031:2110] [node 6] ICS09 handshake done sender: [3:7571519201292333009:2115] self: [3:7571519201292333008:0] peer: [6:7571519203717309144:0] socket: 82 qp: -1
2025-11-11T17:21:29.892776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519201292333031:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:29.892841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:29.906474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519201292332933:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(134525130ns) {HandleHandshakeStatus:329(134521287ns) {DropIncomingHandshake:441(3385ns) DropOutgoingHandshake:456(435ns) SwitchToState:178(1104ns) ProcessPendingSessionEvents:495(1794ns)}}
2025-11-11T17:21:29.918060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519201292332999:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:29.974998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:29.991213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519201292332999:2108] [node 4] ICP22 created new session: [3:7571519201292333034:2108]
2025-11-11T17:21:30.018383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519201292333034:2108] [node 4] ICS09 handshake done sender: [3:7571519201292333011:2116] self: [3:7571519201292333010:0] peer: [4:7571519204853647282:0] socket: 81 qp: -1
2025-11-11T17:21:30.025167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519201292333034:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:30.038607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:30.062755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519201292333007:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.062806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519201292333007:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:30.140813Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519201292333007:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:30.192233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519201292332998:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:30.192286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:30.215270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519201292332998:2109] [node 5] ICP22 created new session: [3:7571519205587300333:2109]
2025-11-11T17:21:30.221878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519205587300333:2109] [node 5] ICS09 handshake done sender: [3:7571519201292333007:2114] self: [3:7571519201292333005:0] peer: [5:7571519209163636959:0] socket: 58 qp: -1
2025-11-11T17:21:30.237906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519205587300333:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:30.246864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519201292332999:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(327927477ns) {HandleHandshakeStatus:329(327922692ns) {DropIncomingHandshake:441(16161ns) DropOutgoingHandshake:456(575ns) SwitchToState:178(926ns) ProcessPendingSessionEvents:495(174736256ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 167818385ns max# 23463120ns}}}
2025-11-11T17:21:30.253330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:30.290208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519201292332998:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(96983085ns) {HandleHandshakeStatus:329(96980217ns) {DropIncomingHandshake:441(13821ns) DropOutgoingHandshake:456(310ns) SwitchToState:178(279ns) ProcessPendingSessionEvents:495(8251993ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8183570ns max# 4921914ns}}}
2025-11-11T17:21:29.931962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519203717309150:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:29.931991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:29.937811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519203717309113:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.937873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:29.937930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519203717309113:2099] [node 5] ICP22 created new session: [6:7571519203717309152:2099]
2025-11-11T17:21:29.937987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519203717309152:2099] [node 5] ICS09 handshake done sender: [6:7571519203717309140:2117] self: [6:7571519203717309143:0] peer: [5:7571519204868669639:0] socket: 96 qp: -1
2025-11-11T17:21:29.938029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519203717309152:2099] [node 5] ICS10 traffic start
2025-11-11T17:21:29.938065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:29.949958Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519203717309139:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:29.953862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519203717309148:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:29.954321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519203717309142:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:29.966792Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:29.982175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519203717309061:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:29.982285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:29.982340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519203717309061:2050] [node 3] ICP22 created new session: [6:7571519203717309154:2050]
2025-11-11T17:21:29.982392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519203717309154:2050] [node 3] ICS09 handshake done sender: [6:7571519203717309139:2116] self: [6:7571519203717309144:0] peer: [3:7571519201292333008:0] socket: 85 qp: -1
2025-11-11T17:21:29.982432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519203717309154:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:29.982468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:29.989576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519203717309060:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(56953456ns) {HandleHandshakeStatus:329(56950479ns) {DropIncomingHandshake:441(673ns) DropOutgoingHandshake:456(4470ns) SwitchToState:178(592ns) ProcessPendingSessionEvents:495(56577684ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56481239ns max# 21335983ns}}}
2025-11-11T17:21:30.015561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519203717309059:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:30.015664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:30.015721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519203717309059:2048] [node 1] ICP22 created new session: [6:7571519208012276453:2048]
2025-11-11T17:21:30.015761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519208012276453:2048] [node 1] ICS09 handshake done sender: [6:7571519203717309142:2119] self: [6:7571519203717309146:0] peer: [1:7571519204638141682:0] socket: 100 qp: -1
2025-11-11T17:21:30.015797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519208012276453:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:30.015859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:30.022911Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519203717309148:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:30.044459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519203717309118:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:30.059029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519203717309059:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(42627439ns) {HandleHandshakeStatus:329(42624608ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(4327ns) SwitchToState:178(180ns) ProcessPendingSessionEvents:495(42191736ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 42103551ns max# 10424937ns}}}
2025-11-11T17:21:30.066732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519203717309061:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(83692207ns) {HandleHandshakeStatus:329(83688361ns) {DropIncomingHandshake:441(1127ns) DropOutgoingHandshake:456(7368ns) SwitchToState:178(730ns) ProcessPendingSessionEvents:495(83203055ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 83082656ns max# 12469554ns}}}
2025-11-11T17:21:30.066837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519203717309061:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:30.090029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519208012276456:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32316
2025-11-11T17:21:30.113924Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519203717309148:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:30.114234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519203717309118:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:30.114341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:30.114398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519203717309118:2102] [node 4] ICP22 created new session: [6:7571519208012276457:2102]
2025-11-11T17:21:30.114443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519208012276457:2102] [node 4] ICS09 handshake done sender: [6:7571519203717309148:2122] self: [6:7571519208012276455:0] peer: [4:7571519204853647265:0] socket: 117 qp: -1
2025-11-11T17:21:30.114487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519208012276457:2102] [node 4] ICS10 traffic start
2025-11-11T17:21:30.114529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:30.169870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:30.170935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519202187528019:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(450135177ns) {HandleHandshakeStatus:329(450132216ns) {DropIncomingHandshake:441(1298ns) DropOutgoingHandshake:456(2352ns) SwitchToState:178(680ns) ProcessPendingSessionEvents:495(403852035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 403620049ns max# 27341747ns}}}
2025-11-11T17:21:30.214226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519202187528019:2109] [node 6] ICP18 handshake (actor [2:7571519202187528036:2117]) is too old
2025-11-11T17:21:30.272252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519202187528020:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(439617803ns) {HandleHandshakeStatus:329(439614743ns) {DropIncomingHandshake:441(727ns) DropOutgoingHandshake:456(680ns) SwitchToState:178(616ns) ProcessPendingSessionEvents:495(360715672ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 360419850ns max# 24050120ns}}}
2025-11-11T17:21:30.273997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519202187528020:2107] [node 4] ICP18 handshake (actor [2:7571519202187528033:2115]) is too old
2025-11-11T17:21:30.279470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519202187528021:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(168824943ns) {HandleHandshakeStatus:329(168821183ns) {DropIncomingHandshake:441(21990ns) DropOutgoingHandshake:456(522ns) SwitchToState:178(707ns) ProcessPendingSessionEvents:495(72416498ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 72317593ns max# 27631107ns}}}
2025-11-11T17:21:30.301120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519204853647256:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(161811944ns) {HandleHandshakeStatus:329(161809250ns) {DropIncomingHandshake:441(15661ns) DropOutgoingHandshake:456(758ns) SwitchToState:178(692ns) ProcessPendingSessionEvents:495(23830248ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23772756ns max# 23255205ns}}}
2025-11-11T17:21:30.363802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:30.363950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519202187528051:2048] [node 1] ICS07 socket disconnect 66 reason# EndOfStream
2025-11-11T17:21:30.363975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519202187528051:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.364036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:30.364069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519202187528051:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:30.367107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:30.367409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519201292333025:2048] [node 1] ICS01 socket: 65 reason# EndOfStream
2025-11-11T17:21:30.367500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519201292332933:2048] [node 1] ICP30 unregister session Session# [3:7571519201292333025:2048] VirtualId# [3:7571519201292333015:0]
2025-11-11T17:21:30.367567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519201292333025:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.367673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:30.367723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:30.369725Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:21:30.373386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:30.371575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519202187527956:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:30.371623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519202187528051:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:30.375175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:30.384305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519204853647289:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-11-11T17:21:30.384379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519204853647193:2048] [node 1] ICP30 unregister session Session# [4:7571519204853647289:2048] VirtualId# [4:7571519204853647279:0]
2025-11-11T17:21:30.384436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519204853647289:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.384540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:30.384596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:30.370714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:30.385442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519208012276453:2048] [node 1] ICS01 socket: 100 reason# EndOfStream
2025-11-11T17:21:30.385482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519203717309059:2048] [node 1] ICP30 unregister session Session# [6:7571519208012276453:2048] VirtualId# [6:7571519203717309146:0]
2025-11-11T17:21:30.385534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519208012276453:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.385850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:30.385910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:30.384335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519204868669658:2048] [node 1] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-11T17:21:30.384385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519204868669658:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.384532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:30.384584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519204868669658:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:30.388268Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address
2025-11-11T17:21:30.389731Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address
2025-11-11T17:21:30.389297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519204868669570:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:30.389896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519204868669658:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:30.398200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519204868669570:2048] [node 1] ICP30 unregister session Session# [5:7571519204868669658:2048] VirtualId# [5:7571519204868669650:0]
2025-11-11T17:21:30.398362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:30.398406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519204868669570:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.398423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.399053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519202187527956:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:30.399117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519202187528051:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:30.399150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519202187527956:2048] [node 1] ICP30 unregister session Session# [2:7571519202187528051:2048] VirtualId# [2:7571519202187528041:0]
2025-11-11T17:21:30.399239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:30.399271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571519202187527956:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.399284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.403913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519204868669570:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:30.408950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571519202187527956:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:30.441060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:30.441287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519201292333026:2049] [node 2] ICS01 socket: 57 reason# EndOfStream
2025-11-11T17:21:30.441330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519201292332934:2049] [node 2] ICP30 unregister session Session# [3:7571519201292333026:2049] VirtualId# [3:7571519201292333021:0]
2025-11-11T17:21:30.441364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519201292333026:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.441459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:30.441503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:30.444237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:30.444471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519204853647281:2049] [node 2] ICS01 socket: 63 reason# EndOfStream
2025-11-11T17:21:30.444509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519204853647194:2049] [node 2] ICP30 unregister session Session# [4:7571519204853647281:2049] VirtualId# [4:7571519204853647274:0]
2025-11-11T17:21:30.444547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519204853647281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.444629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:30.444701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:30.446313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:30.446537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519209163636962:2049] [node 2] ICS01 socket: 105 reason# EndOfStream
2025-11-11T17:21:30.446574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519204868669571:2049] [node 2] ICP30 unregister session Session# [5:7571519209163636962:2049] VirtualId# [5:7571519209163636958:0]
2025-11-11T17:21:30.446622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519209163636962:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.446708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:30.446748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:30.447708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:30.448011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519203717309150:2049] [node 2] ICS07 socket disconnect 72 reason# EndOfStream
2025-11-11T17:21:30.448040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519203717309150:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.448126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:30.448167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519203717309150:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:30.454950Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17052) Couldn't connect to any resolved address
2025-11-11T17:21:30.455299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519203717309060:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17052) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:30.455374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519203717309150:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:30.455422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519203717309060:2049] [node 2] ICP30 unregister session Session# [6:7571519203717309150:2049] VirtualId# [6:7571519203717309138:0]
2025-11-11T17:21:30.455475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:30.455509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519203717309060:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17052) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.455523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17052) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.465642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519203717309060:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:30.508192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:30.508478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519204853647287:2050] [node 3] ICS01 socket: 107 reason# EndOfStream
2025-11-11T17:21:30.508518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519204853647195:2050] [node 3] ICP30 unregister session Session# [4:7571519204853647287:2050] VirtualId# [4:7571519204853647282:0]
2025-11-11T17:21:30.508552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519204853647287:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.508644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:30.508698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:30.509516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:30.517780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:30.518042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519203717309154:2050] [node 3] ICS07 socket disconnect 85 reason# EndOfStream
2025-11-11T17:21:30.518078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519203717309154:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.518195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:30.518235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519203717309154:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:30.522274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519209163636966:2050] [node 3] ICS01 socket: 60 reason# EndOfStream
2025-11-11T17:21:30.522346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519204868669572:2050] [node 3] ICP30 unregister session Session# [5:7571519209163636966:2050] VirtualId# [5:7571519209163636959:0]
2025-11-11T17:21:30.522387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519209163636966:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.522492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:30.522537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:30.527311Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26200) Couldn't connect to any resolved address
2025-11-11T17:21:30.529820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519203717309061:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26200) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:30.538089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519203717309154:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:30.538176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519203717309061:2050] [node 3] ICP30 unregister session Session# [6:7571519203717309154:2050] VirtualId# [6:7571519203717309144:0]
2025-11-11T17:21:30.538237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:30.538282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519203717309061:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.538298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.550061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519203717309061:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:30.601682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:30.601925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519208012276457:2102] [node 4] ICS01 socket: 117 reason# EndOfStream
2025-11-11T17:21:30.601794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:30.602003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519209163636968:2108] [node 4] ICS07 socket disconnect 83 reason# EndOfStream
2025-11-11T17:21:30.602032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519209163636968:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.602118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:30.602161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519209163636968:2108] [node 4] ICS15 start handshake
2025-11-11T17:21:30.602992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519203717309118:2102] [node 4] ICP30 unregister session Session# [6:7571519208012276457:2102] VirtualId# [6:7571519208012276455:0]
2025-11-11T17:21:30.603043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519208012276457:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.603112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:30.603177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:30.609492Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29411) Couldn't connect to any resolved address
2025-11-11T17:21:30.609705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519204868669635:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29411) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:30.609815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519209163636968:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:30.609852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519204868669635:2108] [node 4] ICP30 unregister session Session# [5:7571519209163636968:2108] VirtualId# [5:7571519209163636960:0]
2025-11-11T17:21:30.609959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:30.610032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519204868669635:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29411) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.610050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29411) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:30.620122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519204868669635:2108] [node 4] ICP33 wake up from error state
2025-11-11T17:21:30.719005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:30.719209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519203717309152:2099] [node 5] ICS01 socket: 96 reason# EndOfStream
2025-11-11T17:21:30.719251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519203717309113:2099] [node 5] ICP30 unregister session Session# [6:7571519203717309152:2099] VirtualId# [6:7571519203717309143:0]
2025-11-11T17:21:30.719283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519203717309152:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:30.719414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:30.719472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:30.825152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519208272494340:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:30.825198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519208272494341:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:30.848789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519209514580226:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:30.848832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519209514580227:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:30.870675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519208985938349:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:30.870729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519208985938350:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:30.893309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519207686044076:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:30.893363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519207686044077:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:30.893396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519207686044078:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:30.917319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519209285776645:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:30.917357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519209285776646:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:30.918498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519209285776647:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:30.944197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519205381532547:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:30.944234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519205381532548:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:30.944261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519205381532549:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:31.329334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519209676499875:2080] [node 5] ICP01 ready to work
2025-11-11T17:21:31.393787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519213580743999:2104] [node 6] ICP01 ready to work
2025-11-11T17:21:31.409740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519212567461716:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:31.416239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519213809547597:2121] [node 6] ICP01 ready to work
2025-11-11T17:21:31.481395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519212567461717:2115] [node 6] ICP01 ready to work
2025-11-11T17:21:31.472991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519209676499899:2083] [node 4] ICP01 ready to work
2025-11-11T17:21:31.449423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519213580744004:2101] [node 4] ICP01 ready to work
2025-11-11T17:21:31.513616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519212567461718:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:31.526135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519213809547598:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:31.540636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519213280905737:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:31.565996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519212567461716:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(113970797ns) {Configure:104(113964788ns) {ProcessConfigured:135(52034592ns) {StartInitialHandshake:157(52031324ns) {DropHandshakes:469(2370ns) {DropIncomingHandshake:441(611ns) DropOutgoingHandshake:456(359ns)} GenerateSessionVirtualId:988(1050ns) PrepareNewSessionHandshake:476(8091ns) {SwitchToState:178(4004ns)}}}}}
2025-11-11T17:21:31.546418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519213280905739:2124] [node 5] ICP01 ready to work
2025-11-11T17:21:31.614325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519213809547597:2121] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(197679036ns) {Configure:104(197674557ns) {ProcessConfigured:135(60143574ns) {StartInitialHandshake:157(60140257ns) {DropHandshakes:469(27807806ns) {DropIncomingHandshake:441(950ns) DropOutgoingHandshake:456(2872ns)} GenerateSessionVirtualId:988(3566ns) PrepareNewSessionHandshake:476(4058ns) {SwitchToState:178(1538ns)}}}}}
2025-11-11T17:21:31.630667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519213809547599:2120] [node 5] ICP01 ready to work
2025-11-11T17:21:31.651199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519205381532548:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(218221859ns) {Configure:104(218209015ns) {ProcessConfigured:135(62543ns) {StartInitialHandshake:157(60264ns) {DropHandshakes:469(2174ns) {DropIncomingHandshake:441(377ns) DropOutgoingHandshake:456(780ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:476(2061ns) {SwitchToState:178(620ns)}}}}}
2025-11-11T17:21:31.665903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519213280905739:2124] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(105425622ns) {Configure:104(105422276ns) {ProcessConfigured:135(15997600ns) {StartInitialHandshake:157(15994167ns) {DropHandshakes:469(1542ns) {DropIncomingHandshake:441(318ns) DropOutgoingHandshake:456(331ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:476(5199ns) {SwitchToState:178(1801ns)}}}}}
2025-11-11T17:21:31.631033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519211981011435:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:31.582245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519208272494341:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(131710964ns) {Configure:104(131704500ns) {ProcessConfigured:135(65742ns) {StartInitialHandshake:157(60212ns) {DropHandshakes:469(2606ns) {DropIncomingHandshake:441(550ns) DropOutgoingHandshake:456(501ns)} GenerateSessionVirtualId:988(1034ns) PrepareNewSessionHandshake:476(2213ns) {SwitchToState:178(464ns)}}}}}
2025-11-11T17:21:31.679608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519205381532549:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(122418822ns) {Configure:104(122408682ns) {ProcessConfigured:135(71219ns) {StartInitialHandshake:157(66336ns) {DropHandshakes:469(2684ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(413ns)} GenerateSessionVirtualId:988(701ns) PrepareNewSessionHandshake:476(1384ns) {SwitchToState:178(766ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.686341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519213809547598:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(71521454ns) {Configure:104(71516482ns) {ProcessConfigured:135(58982ns) {StartInitialHandshake:157(56103ns) {DropHandshakes:469(6253ns) {DropIncomingHandshake:441(518ns) DropOutgoingHandshake:456(286ns)} GenerateSessionVirtualId:988(679ns) PrepareNewSessionHandshake:476(1492ns) {SwitchToState:178(516ns)}}}}}
2025-11-11T17:21:31.686490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519213809547601:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47366
2025-11-11T17:21:31.686693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519213809547603:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47366
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.661705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519211981011440:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:31.693650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519213280905747:2118] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.716555Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547601:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.729431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519209676499875:2080] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(354321088ns) {Configure:104(354317206ns) {ProcessConfigured:135(21907349ns) {StartInitialHandshake:157(21904725ns) {DropHandshakes:469(2936ns) {DropIncomingHandshake:441(648ns) DropOutgoingHandshake:456(500ns)} GenerateSessionVirtualId:988(1431ns) PrepareNewSessionHandshake:476(14429ns) {SwitchToState:178(13323ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.728155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519209514580227:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(37632316ns) {Configure:104(37626464ns) {ProcessConfigured:135(24241533ns) {StartInitialHandshake:157(24237894ns) {DropHandshakes:469(1558ns) {DropIncomingHandshake:441(413ns) DropOutgoingHandshake:456(281ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:476(3091ns) {SwitchToState:178(1116ns)}}}}}
2025-11-11T17:21:31.731951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519212567461717:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(156568307ns) {Configure:104(156563255ns) {ProcessConfigured:135(37976541ns) {StartInitialHandshake:157(37973813ns) {DropHandshakes:469(2857ns) {DropIncomingHandshake:441(927ns) DropOutgoingHandshake:456(587ns)} GenerateSessionVirtualId:988(730ns) PrepareNewSessionHandshake:476(4302386ns) {SwitchToState:178(6521ns)}}}}}
2025-11-11T17:21:31.736177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519212567461733:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44597
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.716302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519213280905749:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34073
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.766389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519212567461730:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44597
2025-11-11T17:21:31.681047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519207686044076:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23016617ns) {Configure:104(23012597ns) {ProcessConfigured:135(5209473ns) {StartInitialHandshake:157(5205011ns) {DropHandshakes:469(7553ns) {DropIncomingHandshake:441(448ns) DropOutgoingHandshake:456(535ns)} GenerateSessionVirtualId:988(954ns) PrepareNewSessionHandshake:476(3639ns) {SwitchToState:178(1514ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.783105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519212567461734:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44597
2025-11-11T17:21:31.763080Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547603:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.739647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519209285776645:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(108719571ns) {Configure:104(108713861ns) {ProcessConfigured:135(56895ns) {StartInitialHandshake:157(54740ns) {DropHandshakes:469(2945ns) {DropIncomingHandshake:441(767ns) DropOutgoingHandshake:456(583ns)} GenerateSessionVirtualId:988(1343ns) PrepareNewSessionHandshake:476(1766ns) {SwitchToState:178(482ns)}}}}}
2025-11-11T17:21:31.716485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519213280905750:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34073
2025-11-11T17:21:31.790163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519212567461718:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(46724950ns) {Configure:104(46720008ns) {ProcessConfigured:135(10816964ns) {StartInitialHandshake:157(10813925ns) {DropHandshakes:469(2212ns) {DropIncomingHandshake:441(698ns) DropOutgoingHandshake:456(380ns)} GenerateSessionVirtualId:988(1593ns) PrepareNewSessionHandshake:476(4882ns) {SwitchToState:178(1774ns)}}}}}
2025-11-11T17:21:31.741892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519208985938349:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(276991436ns) {Configure:104(276983274ns) {ProcessConfigured:135(22337426ns) {StartInitialHandshake:157(22335105ns) {DropHandshakes:469(3200ns) {DropIncomingHandshake:441(1225ns) DropOutgoingHandshake:456(316ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:476(4076ns) {SwitchToState:178(986ns)}}}}}
2025-11-11T17:21:31.778253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519205381532547:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(299257572ns) {Configure:104(299252021ns) {ProcessConfigured:135(17554005ns) {StartInitialHandshake:157(14360280ns) {DropHandshakes:469(2841ns) {DropIncomingHandshake:441(383ns) DropOutgoingHandshake:456(504ns)} GenerateSessionVirtualId:988(1461ns) PrepareNewSessionHandshake:476(3785ns) {SwitchToState:178(1635ns)}}}}}
2025-11-11T17:21:31.740264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519213580744008:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:31.710003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011436:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.743152Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519213280905749:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.715789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011441:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
2025-11-11T17:21:31.785867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519213809547607:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47366
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.803197Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519213580744008:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.786051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519209514580226:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.813812Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547608:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.813853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519213809547608:2127] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.815282Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519213809547608:2127] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:31.821078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519213809547615:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47366
2025-11-11T17:21:31.823250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519213809547599:2120] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(135082886ns) {Configure:104(135078318ns) {ProcessConfigured:135(83592ns) {StartInitialHandshake:157(78498ns) {DropHandshakes:469(2937ns) {DropIncomingHandshake:441(746ns) DropOutgoingHandshake:456(556ns)} GenerateSessionVirtualId:988(1269ns) PrepareNewSessionHandshake:476(2270ns) {SwitchToState:178(822ns)}}}}}
2025-11-11T17:21:31.825001Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547607:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.794232Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461730:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.715963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011442:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
2025-11-11T17:21:31.805838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519209285776647:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(314827942ns) {Configure:104(314818302ns) {ProcessConfigured:135(14291654ns) {StartInitialHandshake:157(14288290ns) {DropHandshakes:469(2518ns) {DropIncomingHandshake:441(557ns) DropOutgoingHandshake:456(388ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:476(3131ns) {SwitchToState:178(686ns)}}}}}
2025-11-11T17:21:31.762287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519213280905746:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34073
2025-11-11T17:21:31.823110Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461724:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.823164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519212567461724:2129] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.832878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461722:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.832912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519212567461722:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.834446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461734:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.842942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461720:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.842972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519212567461720:2127] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.844233Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519212567461720:2127] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:31.844555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519208272494341:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:31.844623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:31.844674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519208272494341:2049] [node 3] ICP22 created new session: [1:7571519212567461737:2049]
2025-11-11T17:21:31.844745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519212567461737:2049] [node 3] ICS09 handshake done sender: [1:7571519212567461720:2127] self: [1:7571519212567461719:0] peer: [3:7571519213280905757:0] socket: 65 qp: -1
2025-11-11T17:21:31.844781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519212567461737:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:31.844842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:31.855212Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519212567461722:2128] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.763324Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519213280905750:2142] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.874481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519209676499922:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.758897Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519211981011441:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.816014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519209285776647:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.874847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519209676499921:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:31.875161Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519209676499922:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.875354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519209676499875:2080] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.885305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519209676499924:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:31.888690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519209676499921:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.888845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519205381532547:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.869090Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547615:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.779395Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519211981011442:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.779634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011437:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
2025-11-11T17:21:31.789764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519211981011440:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(31598181ns) {Configure:104(31588507ns) {ProcessConfigured:135(12768071ns) {StartInitialHandshake:157(12764625ns) {DropHandshakes:469(1000719ns) {DropIncomingHandshake:441(659ns) DropOutgoingHandshake:456(944ns)} GenerateSessionVirtualId:988(799ns) PrepareNewSessionHandshake:476(3868ns) {SwitchToState:178(2277ns)}}}}}
2025-11-11T17:21:31.789930Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519211981011436:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.794007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519211981011435:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(66155969ns) {Configure:104(66149383ns) {ProcessConfigured:135(17550284ns) {StartInitialHandshake:157(17548271ns) {DropHandshakes:469(2278ns) {DropIncomingHandshake:441(491ns) DropOutgoingHandshake:456(400ns)} GenerateSessionVirtualId:988(739ns) PrepareNewSessionHandshake:476(4188ns) {SwitchToState:178(1959ns)}}}}}
2025-11-11T17:21:31.803123Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519211981011437:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.806942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519207686044077:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.820038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519207686044076:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.830332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011454:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
2025-11-11T17:21:31.835149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519207686044078:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(118385877ns) {Configure:104(118381630ns) {ProcessConfigured:135(56748ns) {StartInitialHandshake:157(54129ns) {DropHandshakes:469(1806ns) {DropIncomingHandshake:441(340ns) DropOutgoingHandshake:456(512ns)} GenerateSessionVirtualId:988(400ns) PrepareNewSessionHandshake:476(1209ns) {SwitchToState:178(257ns)}}}}}
2025-11-11T17:21:31.835419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519207686044078:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.854587Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519211981011454:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.856159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519211981011448:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.856186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519211981011448:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.865450Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519211981011448:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:31.888725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011461:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
2025-11-11T17:21:31.886685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519211981011435:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:31.925780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:31.925845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519211981011435:2108] [node 5] ICP22 created new session: [4:7571519211981011463:2108]
2025-11-11T17:21:31.925894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519211981011463:2108] [node 5] ICS09 handshake done sender: [4:7571519211981011448:2116] self: [4:7571519211981011443:0] peer: [5:7571519213580744021:0] socket: 92 qp: -1
2025-11-11T17:21:31.926407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519211981011463:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.821950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519213580744004:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(267802544ns) {Configure:104(267793449ns) {ProcessConfigured:135(9252491ns) {StartInitialHandshake:157(9248428ns) {DropHandshakes:469(2529ns) {DropIncomingHandshake:441(663ns) DropOutgoingHandshake:456(480ns)} GenerateSessionVirtualId:988(1180ns) PrepareNewSessionHandshake:476(3172ns) {SwitchToState:178(1444ns)}}}}}
2025-11-11T17:21:31.763520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519208985938350:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.869172Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519212567461724:2129] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.885976Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547610:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.925209Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519209676499924:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.935028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011465:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
2025-11-11T17:21:31.822623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519213580744015:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:31.885785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461733:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.773997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519213280905746:2140] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.936891Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519211981011436:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.965322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519205381532549:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.815042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519213280905756:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34073
2025-11-11T17:21:31.839898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519208985938349:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.849907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519208985938349:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16472047ns) {IncomingHandshake:261(16470487ns) {DropIncomingHandshake:441(1161ns) PrepareNewSessionHandshake:476(3073ns) {SwitchToState:178(1501ns)} GenerateSessionVirtualId:988(641ns) IssueIncomingHandshakeReply:193(9886393ns) {SwitchToState:178(2377ns)}}}
2025-11-11T17:21:31.873997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519213280905758:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34073
2025-11-11T17:21:31.875348Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519213280905746:2140] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:31.876209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519208985938349:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:31.876299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:31.876348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519208985938349:2048] [node 1] ICP22 created new session: [3:7571519213280905759:2048]
2025-11-11T17:21:31.876403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519213280905759:2048] [node 1] ICS09 handshake done sender: [3:7571519213280905746:2140] self: [3:7571519213280905757:0] peer: [1:7571519212567461719:0] socket: 69 qp: -1
2025-11-11T17:21:31.876438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519213280905759:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:31.897803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519209514580227:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:31.885868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519212567461716:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:31.850273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519213580743999:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(188223687ns) {Configure:104(188214307ns) {ProcessConfigured:135(13912796ns) {StartInitialHandshake:157(13910335ns) {DropHandshakes:469(1461ns) {DropIncomingHandshake:441(483ns) DropOutgoingHandshake:456(354ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:476(1772ns) {SwitchToState:178(115ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.876473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:31.938094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519211981011466:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65116
2025-11-11T17:21:31.897890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519213809547610:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.850439Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519213580744015:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.894527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519208272494340:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:31.975380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519209676499927:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:31.987753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519205381532549:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(29551310ns) {IncomingHandshake:261(29548962ns) {DropIncomingHandshake:441(437ns) PrepareNewSessionHandshake:476(9460803ns) {SwitchToState:178(6311ns)} GenerateSessionVirtualId:988(1613ns) IssueIncomingHandshakeReply:193(10342089ns) {SwitchToState:178(3720ns)}}}
2025-11-11T17:21:31.995747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519209676499928:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:32.014152Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519209676499927:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.018859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519209676499930:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:32.029164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519205381532548:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.886149Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519213280905750:2142] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:31.938176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519211981011442:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:31.894633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:31.921443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519213809547622:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47366
2025-11-11T17:21:31.850920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519213580744004:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:32.035621Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6533) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:31.938962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519207686044077:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:31.911604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519213580744024:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.887164Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519213280905744:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.894691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519208272494340:2048] [node 2] ICP22 created new session: [1:7571519212567461744:2048]
2025-11-11T17:21:32.039084Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519209676499928:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.922043Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519213809547610:2128] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:31.939025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.894742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519212567461744:2048] [node 2] ICS09 handshake done sender: [1:7571519212567461724:2129] self: [1:7571519212567461723:0] peer: [2:7571519213809547616:0] socket: 63 qp: -1
2025-11-11T17:21:31.887196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519213280905744:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.917722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519209285776646:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(201531453ns) {Configure:104(201519526ns) {ProcessConfigured:135(22110876ns) {StartInitialHandshake:157(22108012ns) {DropHandshakes:469(6636567ns) {DropIncomingHandshake:441(2225ns) DropOutgoingHandshake:456(1123ns)} GenerateSessionVirtualId:988(887ns) PrepareNewSessionHandshake:476(3800ns) {SwitchToState:178(1590ns)}}}}}
2025-11-11T17:21:31.939076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519207686044077:2049] [node 2] ICP22 created new session: [4:7571519211981011467:2049]
2025-11-11T17:21:32.049782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519205381532549:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6533) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519209676499924:2117] held: no
2025-11-11T17:21:32.085118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519209676499899:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:31.922391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:31.894777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519212567461744:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:31.922441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519209514580227:2049] [node 3] ICP22 created new session: [2:7571519213809547624:2049]
2025-11-11T17:21:31.893646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519213280905741:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.893694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519213280905741:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.894273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519213280905762:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34073
2025-11-11T17:21:31.930244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519208985938350:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:31.935675Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519213280905744:2139] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:31.942761Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:31.957519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519213280905762:2147] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.958957Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519213280905741:2138] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:31.965480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:31.966529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519208985938350:2049] [node 2] ICP22 created new session: [3:7571519213280905766:2049]
2025-11-11T17:21:31.969762Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519213280905752:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.969788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519213280905752:2143] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.970456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519213280905739:2124] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:31.970499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:31.970546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519213280905739:2124] [node 5] ICP22 created new session: [3:7571519213280905767:2124]
2025-11-11T17:21:31.970597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519213280905767:2124] [node 5] ICS09 handshake done sender: [3:7571519213280905744:2139] self: [3:7571519213280905743:0] peer: [5:7571519213580744013:0] socket: 71 qp: -1
2025-11-11T17:21:31.970631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519213280905767:2124] [node 5] ICS10 traffic start
2025-11-11T17:21:31.970669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:31.981842Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519213280905752:2143] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:31.994886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519213280905737:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:31.994966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:31.995015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519213280905737:2108] [node 4] ICP22 created new session: [3:7571519213280905770:2108]
2025-11-11T17:21:31.995088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519213280905770:2108] [node 4] ICS09 handshake done sender: [3:7571519213280905741:2138] self: [3:7571519213280905740:0] peer: [4:7571519211981011457:0] socket: 73 qp: -1
2025-11-11T17:21:31.995131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519213280905770:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:31.995169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:31.996539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519213280905766:2049] [node 2] ICS09 handshake done sender: [3:7571519213280905750:2142] self: [3:7571519213280905755:0] peer: [2:7571519213809547606:0] socket: 79 qp: -1
2025-11-11T17:21:32.004800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519213280905766:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:32.008651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519213280905747:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.008715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:32.008765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519213280905747:2118] [node 6] ICP22 created new session: [3:7571519217575873068:2118]
2025-11-11T17:21:32.008815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519217575873068:2118] [node 6] ICS09 handshake done sender: [3:7571519213280905752:2143] self: [3:7571519213280905751:0] peer: [6:7571519209676499926:0] socket: 108 qp: -1
2025-11-11T17:21:32.008850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519217575873068:2118] [node 6] ICS10 traffic start
2025-11-11T17:21:32.008885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:32.013935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:32.026944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519213280905747:2118] [node 6] ICP35 event processing took too much time PendingConnection:278(17355884ns) {HandleHandshakeStatus:329(17352950ns) {DropIncomingHandshake:441(17522ns) DropOutgoingHandshake:456(329ns) SwitchToState:178(520ns) ProcessPendingSessionEvents:495(16982109ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16815561ns max# 16222097ns}}}
2025-11-11T17:21:32.075253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519208985938350:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(144019568ns) {HandleHandshakeStatus:329(144015590ns) {DropIncomingHandshake:441(1186ns) DropOutgoingHandshake:456(6402ns) SwitchToState:178(3811ns) ProcessPendingSessionEvents:495(35249648ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35187841ns max# 34717460ns}}}
2025-11-11T17:21:32.075368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519208985938350:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:31.939119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519211981011467:2049] [node 2] ICS09 handshake done sender: [4:7571519211981011442:2114] self: [4:7571519211981011451:0] peer: [2:7571519213809547609:0] socket: 82 qp: -1
2025-11-11T17:21:31.926201Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519213580744020:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.926244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519213580744020:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.926401Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:31.945783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519213580744004:2101] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519213580744015:2114] held: no
2025-11-11T17:21:32.018368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519217875711323:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:32.019819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519213580744024:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.019911Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519217875711323:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.021241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519209285776645:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:32.021301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519209285776646:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:32.022042Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519213580744020:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:32.022453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519213580744026:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:32.028749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519213580744023:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:32.042095Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519213580744008:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:32.042356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519213580743999:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.042401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:32.042451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519213580743999:2104] [node 6] ICP22 created new session: [5:7571519217875711328:2104]
2025-11-11T17:21:32.042493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519217875711328:2104] [node 6] ICS09 handshake done sender: [5:7571519213580744020:2116] self: [5:7571519213580744018:0] peer: [6:7571519209676499923:0] socket: 101 qp: -1
2025-11-11T17:21:32.042527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519217875711328:2104] [node 6] ICS10 traffic start
2025-11-11T17:21:32.042565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:32.043673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519217875711327:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:32.050249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519209285776647:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.050320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:32.050365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519209285776647:2050] [node 3] ICP22 created new session: [5:7571519217875711330:2050]
2025-11-11T17:21:32.050975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519217875711323:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:32.051185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519209285776646:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.051248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:32.051313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519209285776646:2049] [node 2] ICP22 created new session: [5:7571519217875711332:2049]
2025-11-11T17:21:32.051358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519217875711332:2049] [node 2] ICS09 handshake done sender: [5:7571519217875711323:2120] self: [5:7571519217875711326:0] peer: [2:7571519213809547617:0] socket: 75 qp: -1
2025-11-11T17:21:32.051405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519217875711332:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:32.051445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: 2025-11-11T17:21:32.065886Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
::ffff:127.1.0.0
2025-11-11T17:21:32.067008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519209285776645:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519213580744024:2118] held: no
2025-11-11T17:21:32.070349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519217875711330:2050] [node 3] ICS09 handshake done sender: [5:7571519213580744008:2111] self: [5:7571519213580744013:0] peer: [3:7571519213280905743:0] socket: 87 qp: -1
2025-11-11T17:21:32.076660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519217875711330:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:32.085030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:32.143269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519209285776647:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(91910165ns) {HandleHandshakeStatus:329(91907295ns) {DropIncomingHandshake:441(776ns) DropOutgoingHandshake:456(4453ns) SwitchToState:178(530ns) ProcessPendingSessionEvents:495(17192394ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17097823ns max# 8377396ns}}}
2025-11-11T17:21:31.894822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:32.086449Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519209676499921:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:31.936693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:31.951971Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461729:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:31.952041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519212567461729:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:31.959639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519212567461716:2107] [node 4] ICP22 created new session: [1:7571519212567461746:2107]
2025-11-11T17:21:31.966313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519212567461746:2107] [node 4] ICS09 handshake done sender: [1:7571519212567461722:2128] self: [1:7571519212567461721:0] peer: [4:7571519211981011453:0] socket: 62 qp: -1
2025-11-11T17:21:31.988737Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519212567461729:2130] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:31.989826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519212567461746:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:31.989885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:32.006931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519212567461717:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.006984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:32.007036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519212567461717:2115] [node 6] ICP22 created new session: [1:7571519216862429045:2115]
2025-11-11T17:21:32.007090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519216862429045:2115] [node 6] ICS09 handshake done sender: [1:7571519212567461729:2130] self: [1:7571519212567461728:0] peer: [6:7571519209676499925:0] socket: 98 qp: -1
2025-11-11T17:21:32.007127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519216862429045:2115] [node 6] ICS10 traffic start
2025-11-11T17:21:32.007174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:32.038766Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519212567461732:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.038811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519212567461732:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:32.054736Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519212567461732:2132] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:32.055015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519212567461718:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.055073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:32.055121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519212567461718:2108] [node 5] ICP22 created new session: [1:7571519216862429048:2108]
2025-11-11T17:21:32.055168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519216862429048:2108] [node 5] ICS09 handshake done sender: [1:7571519212567461732:2132] self: [1:7571519212567461731:0] peer: [5:7571519217875711325:0] socket: 110 qp: -1
2025-11-11T17:21:32.055204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519216862429048:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:32.055241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:32.083445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519212567461716:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(196580410ns) {HandleHandshakeStatus:329(196576731ns) {DropIncomingHandshake:441(659ns) DropOutgoingHandshake:456(608ns) SwitchToState:178(637ns) ProcessPendingSessionEvents:495(53047475ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 52962532ns max# 28019132ns}}}
2025-11-11T17:21:32.083554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519212567461716:2107] [node 4] ICP18 handshake (actor [1:7571519212567461733:2133]) is too old
2025-11-11T17:21:31.942641Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519213809547603:2124] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:31.939154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519211981011467:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:32.093988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519213971467227:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:32.160640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519213580744004:2101] [node 4] ICP35 event processing took too much time PendingConnection:278(30277284ns) {HandleHandshakeStatus:395(30273416ns) {LogHandshakeFail:472(19753ns) {UpdateErrorStateLog:544(5382ns)} DropOutgoingHandshake:456(2985ns) ProcessPendingSessionEvents:495(16543096ns) {unordered DropSessionEvent:512 num# 100 sum# 16512394ns max# 10625722ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:32.102649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519205381532547:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.102710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:32.102769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519205381532547:2048] [node 1] ICP22 created new session: [6:7571519213971467232:2048]
2025-11-11T17:21:32.102821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519213971467232:2048] [node 1] ICS09 handshake done sender: [6:7571519209676499921:2115] self: [6:7571519209676499925:0] peer: [1:7571519212567461728:0] socket: 100 qp: -1
2025-11-11T17:21:32.102860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519213971467232:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:32.102927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:32.104631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519209676499929:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:31.939190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:32.132554Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:32.158050Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:31.942999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:31.965918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519207686044076:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.159429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519209676499899:2083] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519209676499928:2119] held: no
2025-11-11T17:21:32.013796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:32.160349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519213971467230:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:32.016969Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519211981011441:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:32.166517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519205381532548:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519209676499927:2118] held: no
2025-11-11T17:21:32.017305Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6533) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:32.167135Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519209676499922:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:32.022622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519207686044078:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.029502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519207686044076:2048] [node 1] ICP22 created new session: [4:7571519211981011470:2048]
2025-11-11T17:21:32.168629Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519209676499927:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:32.030061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:32.169117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519209676499875:2080] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.169212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:32.030140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519207686044078:2050] [node 3] ICP22 created new session: [4:7571519216275978767:2050]
2025-11-11T17:21:32.169260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519209676499875:2080] [node 5] ICP22 created new session: [6:7571519213971467235:2080]
2025-11-11T17:21:32.030201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519216275978767:2050] [node 3] ICS09 handshake done sender: [4:7571519211981011441:2113] self: [4:7571519211981011457:0] peer: [3:7571519213280905740:0] socket: 81 qp: -1
2025-11-11T17:21:32.169309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519213971467235:2080] [node 5] ICS09 handshake done sender: [6:7571519209676499922:2116] self: [6:7571519209676499923:0] peer: [5:7571519213580744018:0] socket: 106 qp: -1
2025-11-11T17:21:32.169344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519213971467235:2080] [node 5] ICS10 traffic start
2025-11-11T17:21:32.030258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519216275978767:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:32.169379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:32.030316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:32.178150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519205381532548:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.031078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519207686044078:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:32.178252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:32.040295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519211981011470:2048] [node 1] ICS09 handshake done sender: [4:7571519211981011436:2110] self: [4:7571519211981011453:0] peer: [1:7571519212567461721:0] socket: 64 qp: -1
2025-11-11T17:21:32.178311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519205381532548:2049] [node 2] ICP22 created new session: [6:7571519213971467237:2049]
2025-11-11T17:21:32.178379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519213971467237:2049] [node 2] ICS09 handshake done sender: [6:7571519209676499927:2118] self: [6:7571519213971467228:0] peer: [2:7571519213809547600:0] socket: 119 qp: -1
2025-11-11T17:21:32.057735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519211981011470:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:32.178422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519213971467237:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:32.057801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:32.178482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:32.181752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519213971467231:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29432
2025-11-11T17:21:32.086653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519211981011450:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.086677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519211981011450:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:32.087899Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519211981011450:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:32.088156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519211981011440:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.088230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:32.088282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519211981011440:2109] [node 6] ICP22 created new session: [4:7571519216275978771:2109]
2025-11-11T17:21:32.088332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519216275978771:2109] [node 6] ICS09 handshake done sender: [4:7571519211981011450:2117] self: [4:7571519211981011449:0] peer: [6:7571519213971467229:0] socket: 84 qp: -1
2025-11-11T17:21:32.088365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519216275978771:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:32.088400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:31.956256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519213809547598:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.160802Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519213580744015:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:32.129869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519207686044076:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(144812553ns) {HandleHandshakeStatus:329(144810173ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(6682ns) SwitchToState:178(222ns) ProcessPendingSessionEvents:495(16502712ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16436060ns max# 16004813ns}}}
2025-11-11T17:21:32.198459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519211981011435:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(303021240ns) {HandleHandshakeStatus:329(303018857ns) {DropIncomingHandshake:441(23604ns) DropOutgoingHandshake:456(447ns) SwitchToState:178(563ns) ProcessPendingSessionEvents:495(222065378ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 221871906ns max# 29212532ns}}}
2025-11-11T17:21:31.956346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:31.956401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519213809547598:2107] [node 4] ICP22 created new session: [2:7571519213809547626:2107]
2025-11-11T17:21:31.956449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519213809547626:2107] [node 4] ICS09 handshake done sender: [2:7571519213809547610:2128] self: [2:7571519213809547609:0] peer: [4:7571519211981011451:0] socket: 78 qp: -1
2025-11-11T17:21:31.956486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519213809547626:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:31.956565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:31.959358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519213809547624:2049] [node 3] ICS09 handshake done sender: [2:7571519213809547608:2127] self: [2:7571519213809547606:0] peer: [3:7571519213280905755:0] socket: 77 qp: -1
2025-11-11T17:21:31.959800Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:31.965842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519213809547624:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:31.968490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:31.977056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519209514580226:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519213809547603:2124] held: no
2025-11-11T17:21:31.977686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519209514580226:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:31.977737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:31.977788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519209514580226:2048] [node 1] ICP22 created new session: [2:7571519213809547629:2048]
2025-11-11T17:21:31.977847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519213809547629:2048] [node 1] ICS09 handshake done sender: [2:7571519213809547603:2124] self: [2:7571519213809547616:0] peer: [1:7571519212567461723:0] socket: 67 qp: -1
2025-11-11T17:21:31.977884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519213809547629:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:31.977922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:32.019614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519218104514928:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47366
2025-11-11T17:21:32.021745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547618:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.021772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519213809547618:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:32.032311Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519213809547618:2130] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:32.037707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218104514928:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.077800Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519213809547602:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:32.077847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519213809547602:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:32.079617Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519213809547602:2123] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:32.084637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519213809547599:2120] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.098127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:32.105124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519213809547597:2121] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:32.105187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:32.105232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519213809547597:2121] [node 6] ICP22 created new session: [2:7571519218104514933:2121]
2025-11-11T17:21:32.105282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519218104514933:2121] [node 6] ICS09 handshake done sender: [2:7571519213809547602:2123] self: [2:7571519213809547600:0] peer: [6:7571519213971467228:0] socket: 114 qp: -1
2025-11-11T17:21:32.105318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519218104514933:2121] [node 6] ICS10 traffic start
2025-11-11T17:21:32.105360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:32.115355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519213809547599:2120] [node 5] ICP22 created new session: [2:7571519218104514932:2120]
2025-11-11T17:21:32.125955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519218104514932:2120] [node 5] ICS09 handshake done sender: [2:7571519213809547618:2130] self: [2:7571519213809547617:0] peer: [5:7571519217875711326:0] socket: 83 qp: -1
2025-11-11T17:21:32.130951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519209514580227:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(234341083ns) {HandleHandshakeStatus:329(227881664ns) {DropIncomingHandshake:441(18236ns) DropOutgoingHandshake:456(511ns) SwitchToState:178(827ns) ProcessPendingSessionEvents:495(75789408ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75684313ns max# 15452235ns}}}
2025-11-11T17:21:32.155798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519218104514932:2120] [node 5] ICS10 traffic start
2025-11-11T17:21:32.160980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:32.189555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519213809547598:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(75896738ns) {HandleHandshakeStatus:329(75892509ns) {DropIncomingHandshake:441(22358ns) DropOutgoingHandshake:456(364ns) SwitchToState:178(481ns) ProcessPendingSessionEvents:495(75332678ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75222524ns max# 19110278ns}}}
2025-11-11T17:21:32.199655Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519209676499928:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:32.200842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519209676499899:2083] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.200899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:32.200959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519209676499899:2083] [node 4] ICP22 created new session: [6:7571519213971467239:2083]
2025-11-11T17:21:32.201018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519213971467239:2083] [node 4] ICS09 handshake done sender: [6:7571519209676499928:2119] self: [6:7571519213971467229:0] peer: [4:7571519211981011449:0] socket: 121 qp: -1
2025-11-11T17:21:32.201067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519213971467239:2083] [node 4] ICS10 traffic start
2025-11-11T17:21:32.201114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:32.215642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519209676499875:2080] [node 5] ICP35 event processing took too much time PendingConnection:278(21366897ns) {HandleHandshakeStatus:329(21348676ns) {DropIncomingHandshake:441(765ns) DropOutgoingHandshake:456(4044ns) SwitchToState:178(548ns) ProcessPendingSessionEvents:495(20937937ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20861128ns max# 7922760ns}}}
2025-11-11T17:21:32.219112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519213280905737:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(165825386ns) {HandleHandshakeStatus:329(165820752ns) {DropIncomingHandshake:441(16824ns) DropOutgoingHandshake:456(582ns) SwitchToState:178(575ns) ProcessPendingSessionEvents:495(165350225ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 165166211ns max# 14401527ns}}}
2025-11-11T17:21:32.218557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519205381532549:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(136324349ns) {HandleHandshakeStatus:395(136320448ns) {LogHandshakeFail:472(14502ns) {UpdateErrorStateLog:544(5014ns)} DropOutgoingHandshake:456(1234ns) ProcessPendingSessionEvents:495(125766793ns) {unordered DropSessionEvent:512 num# 100 sum# 28985146ns max# 6851683ns}}}
2025-11-11T17:21:32.218709Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519209676499924:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:32.173760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519213580744004:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.177928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:32.178013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519213580744004:2101] [node 4] ICP22 created new session: [5:7571519217875711336:2101]
2025-11-11T17:21:32.178076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519217875711336:2101] [node 4] ICS09 handshake done sender: [5:7571519213580744015:2114] self: [5:7571519213580744021:0] peer: [4:7571519211981011443:0] socket: 93 qp: -1
2025-11-11T17:21:32.178124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519217875711336:2101] [node 4] ICS10 traffic start
2025-11-11T17:21:32.178166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:32.187547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519217875711334:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:32.189700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519217875711338:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6175
2025-11-11T17:21:32.189770Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:32.193087Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519213580744024:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:32.220870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519211981011440:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(102210725ns) {HandleHandshakeStatus:329(102208013ns) {DropIncomingHandshake:441(12107ns) DropOutgoingHandshake:456(437ns) SwitchToState:178(815ns) ProcessPendingSessionEvents:495(101831404ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 101038381ns max# 11723570ns}}}
2025-11-11T17:21:32.223080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519213809547599:2120] [node 5] ICP35 event processing took too much time PendingConnection:278(144425355ns) {HandleHandshakeStatus:329(144422458ns) {DropIncomingHandshake:441(18151ns) DropOutgoingHandshake:456(709ns) SwitchToState:178(785ns) ProcessPendingSessionEvents:495(60997732ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60893713ns max# 14149691ns}}}
2025-11-11T17:21:32.223402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519209285776645:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.223466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:32.224333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519209285776645:2048] [node 1] ICP22 created new session: [5:7571519217875711339:2048]
2025-11-11T17:21:32.224401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519217875711339:2048] [node 1] ICS09 handshake done sender: [5:7571519213580744024:2118] self: [5:7571519217875711325:0] peer: [1:7571519212567461731:0] socket: 112 qp: -1
2025-11-11T17:21:32.224449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519217875711339:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:32.224509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:32.223347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519205381532549:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:32.225483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:32.225552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519205381532549:2050] [node 3] ICP22 created new session: [6:7571519213971467241:2050]
2025-11-11T17:21:32.225655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519213971467241:2050] [node 3] ICS09 handshake done sender: [6:7571519209676499924:2117] self: [6:7571519209676499926:0] peer: [3:7571519213280905751:0] socket: 109 qp: -1
2025-11-11T17:21:32.225688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519213971467241:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:32.225783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:32.297125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:32.297419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519213809547629:2048] [node 1] ICS01 socket: 67 reason# EndOfStream
2025-11-11T17:21:32.297469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519209514580226:2048] [node 1] ICP30 unregister session Session# [2:7571519213809547629:2048] VirtualId# [2:7571519213809547616:0]
2025-11-11T17:21:32.297502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519213809547629:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.297619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:32.297665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:32.299193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:32.299433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519213280905759:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-11-11T17:21:32.299474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519208985938349:2048] [node 1] ICP30 unregister session Session# [3:7571519213280905759:2048] VirtualId# [3:7571519213280905757:0]
2025-11-11T17:21:32.299507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519213280905759:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.299608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:32.299651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:32.302740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:32.303004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519211981011470:2048] [node 1] ICS07 socket disconnect 64 reason# EndOfStream
2025-11-11T17:21:32.303033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519211981011470:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.303119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:32.303159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519211981011470:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:32.305904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:32.306159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519217875711339:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-11-11T17:21:32.306212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519209285776645:2048] [node 1] ICP30 unregister session Session# [5:7571519217875711339:2048] VirtualId# [5:7571519217875711325:0]
2025-11-11T17:21:32.306250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519217875711339:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.306330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:32.306370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:32.306776Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address
2025-11-11T17:21:32.312395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:32.321850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519207686044076:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:32.321949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519211981011470:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:32.321999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519207686044076:2048] [node 1] ICP30 unregister session Session# [4:7571519211981011470:2048] VirtualId# [4:7571519211981011453:0]
2025-11-11T17:21:32.322120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:32.322191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519207686044076:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.322211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.323036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519213971467232:2048] [node 1] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-11T17:21:32.323086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519213971467232:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.323213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:32.323271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519213971467232:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:32.324113Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address
2025-11-11T17:21:32.324443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519205381532547:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:32.324498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519213971467232:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:32.324531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519205381532547:2048] [node 1] ICP30 unregister session Session# [6:7571519213971467232:2048] VirtualId# [6:7571519209676499925:0]
2025-11-11T17:21:32.324650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:32.324705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519205381532547:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.324722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.329398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519207686044076:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:32.332906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519205381532547:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:32.411296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:32.411564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519213280905766:2049] [node 2] ICS01 socket: 79 reason# EndOfStream
2025-11-11T17:21:32.411606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519208985938350:2049] [node 2] ICP30 unregister session Session# [3:7571519213280905766:2049] VirtualId# [3:7571519213280905755:0]
2025-11-11T17:21:32.411642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519213280905766:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.411747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:32.411795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:32.421753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:32.421994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519211981011467:2049] [node 2] ICS01 socket: 82 reason# EndOfStream
2025-11-11T17:21:32.422048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519207686044077:2049] [node 2] ICP30 unregister session Session# [4:7571519211981011467:2049] VirtualId# [4:7571519211981011451:0]
2025-11-11T17:21:32.422080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519211981011467:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.422165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:32.422205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:32.426675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:32.422447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:32.433134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519213971467237:2049] [node 2] ICS01 socket: 119 reason# EndOfStream
2025-11-11T17:21:32.433192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519205381532548:2049] [node 2] ICP30 unregister session Session# [6:7571519213971467237:2049] VirtualId# [6:7571519213971467228:0]
2025-11-11T17:21:32.433229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519213971467237:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.433354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:32.433403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:32.426980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519217875711332:2049] [node 2] ICS01 socket: 75 reason# EndOfStream
2025-11-11T17:21:32.427037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519209285776646:2049] [node 2] ICP30 unregister session Session# [5:7571519217875711332:2049] VirtualId# [5:7571519217875711326:0]
2025-11-11T17:21:32.427079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519217875711332:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.427228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:32.427281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:32.554902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:32.553848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:32.554448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:32.555082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519217875711330:2050] [node 3] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-11T17:21:32.555112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519217875711330:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.554660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519213971467241:2050] [node 3] ICS01 socket: 109 reason# EndOfStream
2025-11-11T17:21:32.554693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519205381532549:2050] [node 3] ICP30 unregister session Session# [6:7571519213971467241:2050] VirtualId# [6:7571519209676499926:0]
2025-11-11T17:21:32.554717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519213971467241:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.554772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:32.554808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:32.555168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:32.555212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519217875711330:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:32.575027Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6533) Couldn't connect to any resolved address
2025-11-11T17:21:32.576921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519209285776647:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6533) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:32.576982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519217875711330:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:32.577019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519209285776647:2050] [node 3] ICP30 unregister session Session# [5:7571519217875711330:2050] VirtualId# [5:7571519213580744013:0]
2025-11-11T17:21:32.577134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:32.577175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519209285776647:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6533) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.577194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6533) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.589739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519209285776647:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:32.677807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:32.678063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519217875711336:2101] [node 4] ICS01 socket: 93 reason# EndOfStream
2025-11-11T17:21:32.678109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519213580744004:2101] [node 4] ICP30 unregister session Session# [5:7571519217875711336:2101] VirtualId# [5:7571519213580744021:0]
2025-11-11T17:21:32.678144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519217875711336:2101] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.678256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:32.678312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:32.679063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:32.679296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519213971467239:2083] [node 4] ICS01 socket: 121 reason# EndOfStream
2025-11-11T17:21:32.679336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519209676499899:2083] [node 4] ICP30 unregister session Session# [6:7571519213971467239:2083] VirtualId# [6:7571519213971467229:0]
2025-11-11T17:21:32.679396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519213971467239:2083] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.679463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:32.679510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:32.737773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:32.738014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519213971467235:2080] [node 5] ICS07 socket disconnect 106 reason# EndOfStream
2025-11-11T17:21:32.738047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519213971467235:2080] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:32.738180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:32.738224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519213971467235:2080] [node 5] ICS15 start handshake
2025-11-11T17:21:32.739205Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7960) Couldn't connect to any resolved address
2025-11-11T17:21:32.739472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519209676499875:2080] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7960) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:32.739519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519213971467235:2080] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:32.739555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519209676499875:2080] [node 5] ICP30 unregister session Session# [6:7571519213971467235:2080] VirtualId# [6:7571519209676499923:0]
2025-11-11T17:21:32.739646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:21:32.739679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519209676499875:2080] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.739694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:32.753710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519209676499875:2080] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:32.850692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519217673420882:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:32.850740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519217673420883:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:32.866969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519214131761329:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:32.867020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519214131761330:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:32.894058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519216581076982:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:32.894115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519216581076983:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:32.911107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519215668845578:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:32.911174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519215668845579:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:32.911204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519215668845580:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:32.938325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519214269602287:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:32.938377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519214269602288:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:32.938415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519214269602289:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:32.960236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519217831098171:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:32.960299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519217831098172:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:32.960344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519217831098173:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:33.075373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519222126065520:2099] [node 5] ICP01 ready to work
2025-11-11T17:21:33.228055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519220876044342:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:33.229656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519215668845579:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(99574231ns) {Configure:104(99568717ns) {ProcessConfigured:135(62272ns) {StartInitialHandshake:157(59793ns) {DropHandshakes:469(3125ns) {DropIncomingHandshake:441(678ns) DropOutgoingHandshake:456(576ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(1640ns) {SwitchToState:178(450ns)}}}}}
2025-11-11T17:21:33.227360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519221968388241:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:33.245612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519214269602287:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(100918769ns) {Configure:104(100913842ns) {ProcessConfigured:135(40013ns) {StartInitialHandshake:157(38447ns) {DropHandshakes:469(1982ns) {DropIncomingHandshake:441(388ns) DropOutgoingHandshake:456(283ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:476(1154ns) {SwitchToState:178(283ns)}}}}}
2025-11-11T17:21:33.246434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519218564569648:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:33.246967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519218564569649:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:33.245200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519220876044343:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:33.258175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519222126065520:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(182210551ns) {Configure:104(182204765ns) {ProcessConfigured:135(95917ns) {StartInitialHandshake:157(92048ns) {DropHandshakes:469(4136ns) {DropIncomingHandshake:441(845ns) DropOutgoingHandshake:456(631ns)} GenerateSessionVirtualId:988(1630ns) PrepareNewSessionHandshake:476(5632ns) {SwitchToState:178(801ns)}}}}}
2025-11-11T17:21:33.258868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519222126065532:2102] [node 4] ICP01 ready to work
2025-11-11T17:21:33.234757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519219963812939:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:33.240442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519218426728708:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:33.241091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519221968388242:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:33.241479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519221968388243:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:33.235229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519219963812940:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:33.270933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519218426728709:2128] [node 6] ICP01 ready to work
2025-11-11T17:21:33.271244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519218426728710:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:33.272554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519220876044344:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:33.298801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519215668845580:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(58571367ns) {Configure:104(58567187ns) {ProcessConfigured:135(15481519ns) {StartInitialHandshake:157(15477751ns) {DropHandshakes:469(2370ns) {DropIncomingHandshake:441(516ns) DropOutgoingHandshake:456(282ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:476(4039ns) {SwitchToState:178(1744ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.313009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812949:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.320159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519220876044355:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17777
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.320785Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519219963812949:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.321895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569661:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.331911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519222126065532:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16844474ns) {Configure:104(16833794ns) {ProcessConfigured:135(53728ns) {StartInitialHandshake:157(51635ns) {DropHandshakes:469(943ns) {DropIncomingHandshake:441(133ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:476(1834ns) {SwitchToState:178(667ns)}}}}}
2025-11-11T17:21:33.333854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519221968388242:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29545454ns) {Configure:104(29542151ns) {ProcessConfigured:135(2982484ns) {StartInitialHandshake:157(2980512ns) {DropHandshakes:469(3341ns) {DropIncomingHandshake:441(892ns) DropOutgoingHandshake:456(627ns)} GenerateSessionVirtualId:988(1094ns) PrepareNewSessionHandshake:476(2050ns) {SwitchToState:178(831ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.354375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812953:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
2025-11-11T17:21:33.372001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519218426728710:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25904252ns) {Configure:104(25898837ns) {ProcessConfigured:135(15780253ns) {StartInitialHandshake:157(15776297ns) {DropHandshakes:469(4344491ns) {DropIncomingHandshake:441(1691ns) DropOutgoingHandshake:456(342ns)} GenerateSessionVirtualId:988(2699ns) PrepareNewSessionHandshake:476(5098ns) {SwitchToState:178(2473ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.374117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519221968388254:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:21:33.373736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519217831098172:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(36066167ns) {Configure:104(36060232ns) {ProcessConfigured:135(61173ns) {StartInitialHandshake:157(52938ns) {DropHandshakes:469(2804ns) {DropIncomingHandshake:441(703ns) DropOutgoingHandshake:456(559ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:476(1712ns) {SwitchToState:178(553ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.380446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519218426728717:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:21:33.380613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519218426728718:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:21:33.381058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519218426728723:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:21:33.374300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519221968388255:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:21:33.374412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519221968388256:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:21:33.377270Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388255:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.376742Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519219963812953:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.377043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812955:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.377287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519215668845580:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.387073Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728717:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.387189Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728718:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.385969Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519218564569661:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.386264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569662:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.386341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519214269602289:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.394391Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519220876044348:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.387051Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388254:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.394450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519220876044348:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.410310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065543:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.395124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519220876044355:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.395569Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519220876044346:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.395589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519220876044346:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.396907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519220876044359:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17777
2025-11-11T17:21:33.396966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519220876044360:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17777
2025-11-11T17:21:33.397890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519220876044348:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:33.404154Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519220876044346:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:33.397408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812957:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
2025-11-11T17:21:33.414313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519220876044343:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.430624Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519222126065543:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.405359Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519219963812953:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:33.449740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569664:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.449843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569665:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.468222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519217831098172:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.414372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:33.405732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519215668845580:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.450300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388256:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.414417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519220876044343:2108] [node 5] ICP22 created new session: [3:7571519220876044364:2108]
2025-11-11T17:21:33.471870Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728714:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.436938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.479442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065547:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.475978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519218564569662:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.414463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519220876044364:2108] [node 5] ICS09 handshake done sender: [3:7571519220876044348:2112] self: [3:7571519220876044347:0] peer: [5:7571519218564569663:0] socket: 75 qp: -1
2025-11-11T17:21:33.439082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519215668845580:2050] [node 3] ICP22 created new session: [4:7571519219963812958:2050]
2025-11-11T17:21:33.485398Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519222126065543:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:33.471949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519218426728714:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.414516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519220876044364:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:33.483526Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519218564569661:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:33.484561Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519218564569664:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.484604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519214269602289:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.484658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:33.484709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519214269602289:2050] [node 3] ICP22 created new session: [5:7571519218564569666:2050]
2025-11-11T17:21:33.484752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519218564569666:2050] [node 3] ICS09 handshake done sender: [5:7571519218564569661:2115] self: [5:7571519218564569663:0] peer: [3:7571519220876044347:0] socket: 76 qp: -1
2025-11-11T17:21:33.485121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519218564569666:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:33.485722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:33.488008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519214269602288:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.474620Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519218426728714:2131] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.450422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519219963812958:2050] [node 3] ICS09 handshake done sender: [4:7571519219963812953:2116] self: [4:7571519219963812956:0] peer: [3:7571519220876044345:0] socket: 77 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.494512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519217831098172:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.414554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:33.473637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519219963812958:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:33.512218Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728716:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.512260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519218426728716:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.512840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519218426728709:2128] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.512920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:33.512973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519218426728709:2128] [node 6] ICP22 created new session: [2:7571519218426728730:2128]
2025-11-11T17:21:33.513022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519218426728730:2128] [node 6] ICS09 handshake done sender: [2:7571519218426728714:2131] self: [2:7571519218426728713:0] peer: [6:7571519222126065546:0] socket: 90 qp: -1
2025-11-11T17:21:33.513057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519218426728730:2128] [node 6] ICS10 traffic start
2025-11-11T17:21:33.513101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:33.494584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:33.426451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519220876044342:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.535140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519221968388262:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:21:33.494631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519217831098172:2049] [node 2] ICP22 created new session: [6:7571519222126065548:2049]
2025-11-11T17:21:33.494677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519222126065548:2049] [node 2] ICS09 handshake done sender: [6:7571519222126065543:2115] self: [6:7571519222126065546:0] peer: [2:7571519218426728713:0] socket: 92 qp: -1
2025-11-11T17:21:33.494717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519222126065548:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:33.494751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:33.496476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065550:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.519002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065552:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.522673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519222126065552:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.522804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519217831098173:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.537186Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519222126065550:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.426548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:33.426602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519220876044342:2107] [node 4] ICP22 created new session: [3:7571519220876044366:2107]
2025-11-11T17:21:33.426658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519220876044366:2107] [node 4] ICS09 handshake done sender: [3:7571519220876044346:2111] self: [3:7571519220876044345:0] peer: [4:7571519219963812956:0] socket: 74 qp: -1
2025-11-11T17:21:33.426693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519220876044366:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:33.426760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:33.501997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519220876044360:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.504097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519216581076983:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.504364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519220876044370:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17777
2025-11-11T17:21:33.522638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519220876044370:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.532850Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519220876044352:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.532875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519220876044352:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.533663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519220876044352:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:33.473736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:33.499636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519215668845580:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(93039883ns) {HandleHandshakeStatus:329(93037409ns) {DropIncomingHandshake:441(1267ns) DropOutgoingHandshake:456(6354ns) SwitchToState:178(480ns) ProcessPendingSessionEvents:495(23173412ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22027758ns max# 7384567ns}}}
2025-11-11T17:21:33.511212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
2025-11-11T17:21:33.523282Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519219963812960:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.542217Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519219963812955:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.542432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519215668845578:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.542992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065554:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.550021Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388249:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.551877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519220876044359:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.551943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519220876044344:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.552053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:33.552107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519220876044344:2109] [node 6] ICP22 created new session: [3:7571519220876044374:2109]
2025-11-11T17:21:33.552153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519220876044374:2109] [node 6] ICS09 handshake done sender: [3:7571519220876044352:2114] self: [3:7571519220876044351:0] peer: [6:7571519222126065553:0] socket: 110 qp: -1
2025-11-11T17:21:33.552196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519220876044374:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:33.552229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:33.555284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519218426728709:2128] [node 6] ICP35 event processing took too much time PendingConnection:278(41515252ns) {HandleHandshakeStatus:329(41512065ns) {DropIncomingHandshake:441(427ns) DropOutgoingHandshake:456(603ns) SwitchToState:178(532ns) ProcessPendingSessionEvents:495(41137169ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39220659ns max# 12732279ns}}}
2025-11-11T17:21:33.555581Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728712:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.555604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519218426728712:2130] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.556995Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728723:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.558497Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28997) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:33.550071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519221968388249:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.558042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812964:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
2025-11-11T17:21:33.560696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519216581076982:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.550826Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388262:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.553954Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519221968388249:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.561143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519221968388241:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.566330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519218426728732:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:21:33.566512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519214131761329:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.567657Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:33.561207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:33.561258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519221968388241:2107] [node 4] ICP22 created new session: [1:7571519221968388266:2107]
2025-11-11T17:21:33.561300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519221968388266:2107] [node 4] ICS09 handshake done sender: [1:7571519221968388249:2112] self: [1:7571519221968388248:0] peer: [4:7571519219963812963:0] socket: 86 qp: -1
2025-11-11T17:21:33.561338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519221968388266:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:33.561374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:33.562713Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388245:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.562737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519221968388245:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.564538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519222126065520:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.569780Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388247:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.569830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519221968388247:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.564630Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519222126065552:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:33.568982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.577976Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519221968388245:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:33.582280Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519219963812955:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:33.582861Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519221968388247:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.583218Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519218564569657:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.583253Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519218564569657:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.584034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.584806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519215668845578:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519219963812955:2117] held: no
2025-11-11T17:21:33.585405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519215668845578:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.585485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:33.585529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519215668845578:2048] [node 1] ICP22 created new session: [4:7571519219963812966:2048]
2025-11-11T17:21:33.585582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519219963812966:2048] [node 1] ICS09 handshake done sender: [4:7571519219963812955:2117] self: [4:7571519219963812963:0] peer: [1:7571519221968388248:0] socket: 87 qp: -1
2025-11-11T17:21:33.585631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519219963812966:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:33.585665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:33.587517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812965:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
2025-11-11T17:21:33.568617Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519218426728716:2132] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:33.578243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519217831098173:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28997) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519222126065552:2118] held: no
2025-11-11T17:21:33.578971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519217831098173:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.579024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:33.579076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519217831098173:2050] [node 3] ICP22 created new session: [6:7571519222126065556:2050]
2025-11-11T17:21:33.579149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519222126065556:2050] [node 3] ICS09 handshake done sender: [6:7571519222126065552:2118] self: [6:7571519222126065553:0] peer: [3:7571519220876044351:0] socket: 112 qp: -1
2025-11-11T17:21:33.579196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519222126065556:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:33.579236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.589995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065558:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.594491Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519218564569671:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.589649Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519218426728712:2130] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:33.598151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519217673420883:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.598235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:33.598290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519217673420883:2049] [node 3] ICP22 created new session: [1:7571519221968388271:2049]
2025-11-11T17:21:33.598339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519221968388271:2049] [node 3] ICS09 handshake done sender: [1:7571519221968388245:2110] self: [1:7571519221968388244:0] peer: [3:7571519220876044376:0] socket: 84 qp: -1
2025-11-11T17:21:33.598377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519221968388271:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:33.598410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:33.599690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519217673420882:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.599723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:33.599761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519217673420882:2048] [node 2] ICP22 created new session: [1:7571519221968388273:2048]
2025-11-11T17:21:33.599798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519221968388273:2048] [node 2] ICS09 handshake done sender: [1:7571519221968388247:2111] self: [1:7571519221968388246:0] peer: [2:7571519218426728736:0] socket: 85 qp: -1
2025-11-11T17:21:33.599823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519221968388273:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:33.599854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.604169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065559:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.604227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065560:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.605065Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:33.603550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569673:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.615166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519222126065558:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.604550Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519218564569662:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:33.610723Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519218564569657:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:33.610156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728732:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.612408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519218426728710:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.612467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:33.612520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519218426728710:2109] [node 5] ICP22 created new session: [2:7571519218426728738:2109]
2025-11-11T17:21:33.612569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519218426728738:2109] [node 5] ICS09 handshake done sender: [2:7571519218426728716:2132] self: [2:7571519218426728715:0] peer: [5:7571519218564569668:0] socket: 91 qp: -1
2025-11-11T17:21:33.612608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519218426728738:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:33.612647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:33.613646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519214131761330:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.613678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:33.613722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519214131761330:2049] [node 3] ICP22 created new session: [2:7571519218426728740:2049]
2025-11-11T17:21:33.613770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519218426728740:2049] [node 3] ICS09 handshake done sender: [2:7571519218426728712:2130] self: [2:7571519218426728711:0] peer: [3:7571519220876044371:0] socket: 89 qp: -1
2025-11-11T17:21:33.613808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519218426728740:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:33.613834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.616150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519220876044377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17777
2025-11-11T17:21:33.616571Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519218564569673:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.616614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519214269602288:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.616655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:33.616695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519214269602288:2049] [node 2] ICP22 created new session: [5:7571519218564569675:2049]
2025-11-11T17:21:33.616734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519218564569675:2049] [node 2] ICS09 handshake done sender: [5:7571519218564569662:2116] self: [5:7571519218564569668:0] peer: [2:7571519218426728715:0] socket: 93 qp: -1
2025-11-11T17:21:33.616764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519218564569675:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:33.616790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:33.618206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519218564569648:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.618255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:33.618300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519218564569648:2108] [node 6] ICP22 created new session: [5:7571519218564569677:2108]
2025-11-11T17:21:33.618348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519218564569677:2108] [node 6] ICS09 handshake done sender: [5:7571519218564569657:2114] self: [5:7571519218564569656:0] peer: [6:7571519222126065555:0] socket: 107 qp: -1
2025-11-11T17:21:33.618379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519218564569677:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:33.618407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.620756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519216581076982:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519220876044359:2117] held: no
2025-11-11T17:21:33.621652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519220876044378:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17777
2025-11-11T17:21:33.622322Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519222126065550:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:33.622668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519222126065532:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.622717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519222126065520:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.622754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:33.622792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519222126065520:2099] [node 5] ICP22 created new session: [6:7571519222126065562:2099]
2025-11-11T17:21:33.622833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519222126065562:2099] [node 5] ICS09 handshake done sender: [6:7571519222126065550:2117] self: [6:7571519222126065555:0] peer: [5:7571519218564569656:0] socket: 73 qp: -1
2025-11-11T17:21:33.622863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519222126065562:2099] [node 5] ICS10 traffic start
2025-11-11T17:21:33.622894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:33.626754Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519222126065560:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.626954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519217831098171:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.629418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519214131761330:2049] [node 3] ICP18 handshake (actor [2:7571519218426728732:2138]) is too old
2025-11-11T17:21:33.629548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519218426728737:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15602
2025-11-11T17:21:33.634967Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519220876044360:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:33.633690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519219963812948:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.636901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519218564569649:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.639011Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388253:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.639067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519221968388253:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.645116Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519221968388253:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.642033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065565:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.650445Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519218426728723:2137] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:33.650763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519214131761329:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.650826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:33.655185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519214131761329:2048] [node 1] ICP22 created new session: [2:7571519218426728743:2048]
2025-11-11T17:21:33.655257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519218426728743:2048] [node 1] ICS09 handshake done sender: [2:7571519218426728723:2137] self: [2:7571519218426728736:0] peer: [1:7571519221968388246:0] socket: 88 qp: -1
2025-11-11T17:21:33.655300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519218426728743:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:33.655355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:33.633768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519219963812948:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.657859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519216581076983:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.657541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222126065566:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49277
2025-11-11T17:21:33.678341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:33.640580Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519219963812948:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:33.681323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519216581076983:2049] [node 2] ICP22 created new session: [3:7571519220876044379:2049]
2025-11-11T17:21:33.668428Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519222126065558:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:33.686782Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519222126065560:2122] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:33.687086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519222126065532:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.687139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:33.687186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519222126065532:2102] [node 4] ICP22 created new session: [6:7571519222126065567:2102]
2025-11-11T17:21:33.687229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519222126065567:2102] [node 4] ICS09 handshake done sender: [6:7571519222126065558:2120] self: [6:7571519222126065561:0] peer: [4:7571519219963812947:0] socket: 127 qp: -1
2025-11-11T17:21:33.687261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519222126065567:2102] [node 4] ICS10 traffic start
2025-11-11T17:21:33.687294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:33.681369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519220876044379:2049] [node 2] ICS09 handshake done sender: [3:7571519220876044360:2118] self: [3:7571519220876044371:0] peer: [2:7571519218426728711:0] socket: 97 qp: -1
2025-11-11T17:21:33.690145Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:33.681401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519220876044379:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:33.681439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:33.681503Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519219963812965:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.693837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519221968388243:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.693909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:33.693958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519221968388243:2109] [node 6] ICP22 created new session: [1:7571519221968388278:2109]
2025-11-11T17:21:33.694019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519221968388278:2109] [node 6] ICS09 handshake done sender: [1:7571519221968388253:2114] self: [1:7571519221968388252:0] peer: [6:7571519222126065564:0] socket: 81 qp: -1
2025-11-11T17:21:33.694060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519221968388278:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:33.694090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:33.695543Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519221968388251:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.695565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519221968388251:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.704705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519216581076983:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(46257947ns) {HandleHandshakeStatus:329(46255128ns) {DropIncomingHandshake:441(805ns) DropOutgoingHandshake:456(5002ns) SwitchToState:178(460ns) ProcessPendingSessionEvents:495(22270196ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19144424ns max# 8488685ns}}}
2025-11-11T17:21:33.681562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519219963812940:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.692212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519214269602287:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.681672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:33.706406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519217831098171:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.705265Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:33.681726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519219963812940:2109] [node 6] ICP22 created new session: [4:7571519219963812970:2109]
2025-11-11T17:21:33.684766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519219963812970:2109] [node 6] ICS09 handshake done sender: [4:7571519219963812948:2114] self: [4:7571519219963812947:0] peer: [6:7571519222126065561:0] socket: 126 qp: -1
2025-11-11T17:21:33.684825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519219963812970:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:33.684879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:33.707038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519219963812946:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.707066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519219963812946:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.707729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519215668845579:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:33.712447Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519220876044359:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:33.706475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.712689Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519218426728720:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:33.712741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519218426728720:2135] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:33.708546Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519219963812946:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:33.713492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569681:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.718584Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519216581076983:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:33.713959Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519218564569671:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:33.714158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519218564569649:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.714202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:33.714241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519218564569649:2109] [node 4] ICP22 created new session: [5:7571519218564569682:2109]
2025-11-11T17:21:33.714290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519218564569682:2109] [node 4] ICS09 handshake done sender: [5:7571519218564569671:2119] self: [5:7571519218564569679:0] peer: [4:7571519219963812945:0] socket: 116 qp: -1
2025-11-11T17:21:33.714326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519218564569682:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:33.714404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:33.708833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519217831098171:2048] [node 1] ICP22 created new session: [6:7571519222126065569:2048]
2025-11-11T17:21:33.708888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519222126065569:2048] [node 1] ICS09 handshake done sender: [6:7571519222126065560:2122] self: [6:7571519222126065564:0] peer: [1:7571519221968388252:0] socket: 130 qp: -1
2025-11-11T17:21:33.708930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519222126065569:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:33.708977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:33.710903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519222126065532:2102] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:33.711328Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:33.717788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519219963812939:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.728043Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519218426728720:2135] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:33.729190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519217831098171:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:33.725516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519216581076982:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.717839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:33.717868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519219963812939:2108] [node 5] ICP22 created new session: [4:7571519219963812974:2108]
2025-11-11T17:21:33.717903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519219963812974:2108] [node 5] ICS09 handshake done sender: [4:7571519219963812946:2113] self: [4:7571519219963812945:0] peer: [5:7571519218564569679:0] socket: 113 qp: -1
2025-11-11T17:21:33.717932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519219963812974:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:33.717961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:33.737206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519219963812939:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(18578106ns) {HandleHandshakeStatus:329(18570938ns) {DropIncomingHandshake:441(11834ns) DropOutgoingHandshake:456(462ns) SwitchToState:178(320ns) ProcessPendingSessionEvents:495(18285873ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18207712ns max# 5785537ns}}}
2025-11-11T17:21:33.735493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519218426728708:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.735585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:33.735627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519218426728708:2107] [node 4] ICP22 created new session: [2:7571519218426728746:2107]
2025-11-11T17:21:33.735671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519218426728746:2107] [node 4] ICS09 handshake done sender: [2:7571519218426728720:2135] self: [2:7571519218426728719:0] peer: [4:7571519219963812972:0] socket: 109 qp: -1
2025-11-11T17:21:33.735708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519218426728746:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:33.735737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:33.733657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519221968388277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-11T17:21:33.734304Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:33.725626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:33.725760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519216581076982:2048] [node 1] ICP22 created new session: [3:7571519220876044381:2048]
2025-11-11T17:21:33.728761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519220876044381:2048] [node 1] ICS09 handshake done sender: [3:7571519220876044359:2117] self: [3:7571519220876044376:0] peer: [1:7571519221968388244:0] socket: 96 qp: -1
2025-11-11T17:21:33.733765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519220876044381:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:33.733834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.739724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519219963812976:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26895
2025-11-11T17:21:33.755919Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519221968388251:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:33.756721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519221968388242:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:33.756789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:33.756835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519221968388242:2108] [node 5] ICP22 created new session: [1:7571519221968388281:2108]
2025-11-11T17:21:33.756881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519221968388281:2108] [node 5] ICS09 handshake done sender: [1:7571519221968388251:2113] self: [1:7571519221968388250:0] peer: [5:7571519218564569680:0] socket: 122 qp: -1
2025-11-11T17:21:33.756924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519221968388281:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:33.756958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:33.769856Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:33.778529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519218564569684:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63280
2025-11-11T17:21:33.779305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519215668845579:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519219963812965:2121] held: no
2025-11-11T17:21:33.784964Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:33.790118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519214269602287:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519218564569673:2121] held: no
2025-11-11T17:21:33.790753Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519218564569673:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:33.791305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519214269602287:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.791350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:33.791411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519214269602287:2048] [node 1] ICP22 created new session: [5:7571519218564569685:2048]
2025-11-11T17:21:33.791455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519218564569685:2048] [node 1] ICS09 handshake done sender: [5:7571519218564569673:2121] self: [5:7571519218564569680:0] peer: [1:7571519221968388250:0] socket: 124 qp: -1
2025-11-11T17:21:33.791498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519218564569685:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:33.791536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:33.844203Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519219963812965:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:33.845513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519215668845579:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:33.845571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:33.845637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519215668845579:2049] [node 2] ICP22 created new session: [4:7571519219963812977:2049]
2025-11-11T17:21:33.845690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519219963812977:2049] [node 2] ICS09 handshake done sender: [4:7571519219963812965:2121] self: [4:7571519219963812972:0] peer: [2:7571519218426728719:0] socket: 125 qp: -1
2025-11-11T17:21:33.845725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519219963812977:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:33.845757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:33.902847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:33.903106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519218426728743:2048] [node 1] ICS07 socket disconnect 88 reason# EndOfStream
2025-11-11T17:21:33.903143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519218426728743:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.903259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:33.903302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519218426728743:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:33.919866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:33.920131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519218564569685:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-11-11T17:21:33.920196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519214269602287:2048] [node 1] ICP30 unregister session Session# [5:7571519218564569685:2048] VirtualId# [5:7571519218564569680:0]
2025-11-11T17:21:33.920231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519218564569685:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.920308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:33.920346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:33.917848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:33.918456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:33.931846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519219963812966:2048] [node 1] ICS01 socket: 87 reason# EndOfStream
2025-11-11T17:21:33.933341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519215668845578:2048] [node 1] ICP30 unregister session Session# [4:7571519219963812966:2048] VirtualId# [4:7571519219963812963:0]
2025-11-11T17:21:33.933390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519219963812966:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.937647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:33.937782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:33.937817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519220876044381:2048] [node 1] ICS01 socket: 96 reason# EndOfStream
2025-11-11T17:21:33.933261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:33.938927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519216581076982:2048] [node 1] ICP30 unregister session Session# [3:7571519220876044381:2048] VirtualId# [3:7571519220876044376:0]
2025-11-11T17:21:33.938986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519220876044381:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.939057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:33.939100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:33.938636Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:21:33.938004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519222126065569:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-11T17:21:33.938041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519217831098171:2048] [node 1] ICP30 unregister session Session# [6:7571519222126065569:2048] VirtualId# [6:7571519222126065564:0]
2025-11-11T17:21:33.938079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519222126065569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.938146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:33.938183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:33.979338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:33.979666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519220876044379:2049] [node 2] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-11T17:21:33.979703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519220876044379:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.979808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:33.979863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519220876044379:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:33.981020Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Couldn't connect to any resolved address
2025-11-11T17:21:33.981340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519216581076983:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:33.981389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519220876044379:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:33.981428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519216581076983:2049] [node 2] ICP30 unregister session Session# [3:7571519220876044379:2049] VirtualId# [3:7571519220876044371:0]
2025-11-11T17:21:33.981532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:33.981564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519216581076983:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:33.981581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62012) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:33.988239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:33.988502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519219963812977:2049] [node 2] ICS01 socket: 125 reason# EndOfStream
2025-11-11T17:21:33.988545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519215668845579:2049] [node 2] ICP30 unregister session Session# [4:7571519219963812977:2049] VirtualId# [4:7571519219963812972:0]
2025-11-11T17:21:33.988587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519219963812977:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.988685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:33.988826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:33.990908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519216581076983:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:33.997852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:33.998110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519222126065548:2049] [node 2] ICS01 socket: 92 reason# EndOfStream
2025-11-11T17:21:33.997842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:33.998131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519218564569675:2049] [node 2] ICS01 socket: 93 reason# EndOfStream
2025-11-11T17:21:33.998177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519214269602288:2049] [node 2] ICP30 unregister session Session# [5:7571519218564569675:2049] VirtualId# [5:7571519218564569668:0]
2025-11-11T17:21:33.998213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519218564569675:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.998328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:33.998381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:33.998151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519217831098172:2049] [node 2] ICP30 unregister session Session# [6:7571519222126065548:2049] VirtualId# [6:7571519222126065546:0]
2025-11-11T17:21:33.998190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519222126065548:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:33.998269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:33.998314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:34.064772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:34.065067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519218564569666:2050] [node 3] ICS01 socket: 76 reason# EndOfStream
2025-11-11T17:21:34.065123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519214269602289:2050] [node 3] ICP30 unregister session Session# [5:7571519218564569666:2050] VirtualId# [5:7571519218564569663:0]
2025-11-11T17:21:34.065158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519218564569666:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:34.065261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:34.065310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:34.067598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:34.067888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519219963812958:2050] [node 3] ICS07 socket disconnect 77 reason# EndOfStream
2025-11-11T17:21:34.067935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519219963812958:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:34.068053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:34.068146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519219963812958:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:34.066021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:34.071478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519222126065556:2050] [node 3] ICS01 socket: 112 reason# EndOfStream
2025-11-11T17:21:34.071533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519217831098173:2050] [node 3] ICP30 unregister session Session# [6:7571519222126065556:2050] VirtualId# [6:7571519222126065553:0]
2025-11-11T17:21:34.071581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519222126065556:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:34.071709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:34.071761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:34.072899Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28997) Couldn't connect to any resolved address
2025-11-11T17:21:34.073145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519215668845580:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28997) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:34.073205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519219963812958:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:34.073242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519215668845580:2050] [node 3] ICP30 unregister session Session# [4:7571519219963812958:2050] VirtualId# [4:7571519219963812956:0]
2025-11-11T17:21:34.073328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:34.073362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519215668845580:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:34.073377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:34.109818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:34.110050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519218564569682:2109] [node 4] ICS01 socket: 116 reason# EndOfStream
2025-11-11T17:21:34.110101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519218564569649:2109] [node 4] ICP30 unregister session Session# [5:7571519218564569682:2109] VirtualId# [5:7571519218564569679:0]
2025-11-11T17:21:34.110139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519218564569682:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:34.110246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:34.110294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:34.118654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:34.118901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519222126065567:2102] [node 4] ICS07 socket disconnect 127 reason# EndOfStream
2025-11-11T17:21:34.118936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519222126065567:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:34.119064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:34.119119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519222126065567:2102] [node 4] ICS15 start handshake
2025-11-11T17:21:34.122020Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3945) Couldn't connect to any resolved address
2025-11-11T17:21:34.122282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519222126065532:2102] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3945) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:34.122346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519222126065567:2102] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:34.122380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519222126065532:2102] [node 4] ICP30 unregister session Session# [6:7571519222126065567:2102] VirtualId# [6:7571519222126065561:0]
2025-11-11T17:21:34.122423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:34.122458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519222126065532:2102] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:34.122472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:34.129729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519222126065532:2102] [node 4] ICP33 wake up from error state
2025-11-11T17:21:34.197763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:34.198088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519222126065562:2099] [node 5] ICS01 socket: 73 reason# EndOfStream
2025-11-11T17:21:34.198170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519222126065520:2099] [node 5] ICP30 unregister session Session# [6:7571519222126065562:2099] VirtualId# [6:7571519222126065555:0]
2025-11-11T17:21:34.198225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519222126065562:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:34.198364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:34.198444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:34.293944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519223433694856:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:34.293984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519223433694857:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:34.318081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519223680860108:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:34.318121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519223680860109:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:34.341801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519224994065958:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:34.341856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519224994065959:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:34.364890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519222578991926:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:34.364929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519222578991927:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:34.364960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519222578991928:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:34.405851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519224403942935:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:34.405894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519224403942936:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:34.405922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519224403942937:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:34.431251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519222554164881:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:34.431352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519222554164883:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:34.431394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519222554164882:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:34.669304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519224403942997:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:34.676096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519222554164946:2104] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:34.696607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519223433694922:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:34.697080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519223433694923:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:34.697361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519223433694924:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:34.699447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519223433694916:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-11T17:21:34.702195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519224403942998:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:34.734202Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519224403942998:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:34.737491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519223433694856:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(52135719ns) {Configure:104(52127049ns) {ProcessConfigured:135(602134ns) {StartInitialHandshake:157(597882ns) {DropHandshakes:469(3639ns) {DropIncomingHandshake:441(1079ns) DropOutgoingHandshake:456(846ns)} GenerateSessionVirtualId:988(2087ns) PrepareNewSessionHandshake:476(3742ns) {SwitchToState:178(1474ns)}}}}}
2025-11-11T17:21:34.737555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519223680860172:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:34.738536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519223680860108:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(30491523ns) {Configure:104(30483115ns) {ProcessConfigured:135(44335ns) {StartInitialHandshake:157(42453ns) {DropHandshakes:469(1474ns) {DropIncomingHandshake:441(432ns) DropOutgoingHandshake:456(167ns)} GenerateSessionVirtualId:988(326ns) PrepareNewSessionHandshake:476(1091ns) {SwitchToState:178(250ns)}}}}}
2025-11-11T17:21:34.781692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519224994066023:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:34.782186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519224994066025:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:34.782582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519224994066026:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:34.738930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519223680860175:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:34.758311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519224403942997:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(72456487ns) {Configure:104(72453623ns) {ProcessConfigured:135(52063ns) {StartInitialHandshake:157(50139ns) {DropHandshakes:469(1203ns) {DropIncomingHandshake:441(285ns) DropOutgoingHandshake:456(384ns)} GenerateSessionVirtualId:988(281ns) PrepareNewSessionHandshake:476(3005ns) {SwitchToState:178(156ns)}}}}}
2025-11-11T17:21:34.739351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519223680860176:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:34.862127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519222578991927:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(45695529ns) {Configure:104(45691203ns) {ProcessConfigured:135(92426ns) {StartInitialHandshake:157(87134ns) {DropHandshakes:469(2529ns) {DropIncomingHandshake:441(484ns) DropOutgoingHandshake:456(344ns)} GenerateSessionVirtualId:988(1079ns) PrepareNewSessionHandshake:476(1833ns) {SwitchToState:178(808ns)}}}}}
2025-11-11T17:21:34.862674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519222578991993:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:34.897689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519223433694923:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(98200829ns) {Configure:104(98190613ns) {ProcessConfigured:135(23485150ns) {StartInitialHandshake:157(23481136ns) {DropHandshakes:469(3184ns) {DropIncomingHandshake:441(732ns) DropOutgoingHandshake:456(671ns)} GenerateSessionVirtualId:988(1743ns) PrepareNewSessionHandshake:476(8435ns) {SwitchToState:178(5443ns)}}}}}
2025-11-11T17:21:34.853692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519223680860176:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(55714492ns) {Configure:104(55698968ns) {ProcessConfigured:135(73658ns) {StartInitialHandshake:157(71345ns) {DropHandshakes:469(2297ns) {DropIncomingHandshake:441(580ns) DropOutgoingHandshake:456(408ns)} GenerateSessionVirtualId:988(1178ns) PrepareNewSessionHandshake:476(1543ns) {SwitchToState:178(513ns)}}}}}
2025-11-11T17:21:34.897790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519224403943004:2111] [node 4] ICP01 ready to work
2025-11-11T17:21:34.926645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519222578991928:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55667847ns) {Configure:104(55663181ns) {ProcessConfigured:135(18335485ns) {StartInitialHandshake:157(18332186ns) {DropHandshakes:469(3260ns) {DropIncomingHandshake:441(932ns) DropOutgoingHandshake:456(537ns)} GenerateSessionVirtualId:988(1174ns) PrepareNewSessionHandshake:476(4238ns) {SwitchToState:178(2390ns)}}}}}
2025-11-11T17:21:34.897416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519223680860175:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(88735615ns) {Configure:104(88732267ns) {ProcessConfigured:135(35484250ns) {StartInitialHandshake:157(35479881ns) {DropHandshakes:469(3363ns) {DropIncomingHandshake:441(788ns) DropOutgoingHandshake:456(698ns)} GenerateSessionVirtualId:988(862ns) PrepareNewSessionHandshake:476(7018ns) {SwitchToState:178(3498ns)}}}}}
2025-11-11T17:21:34.942016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519224994065959:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(204811792ns) {Configure:104(204807986ns) {ProcessConfigured:135(10568919ns) {StartInitialHandshake:157(10563517ns) {DropHandshakes:469(10467197ns) {DropIncomingHandshake:441(1175ns) DropOutgoingHandshake:456(3814ns)} GenerateSessionVirtualId:988(3637ns) PrepareNewSessionHandshake:476(3963ns) {SwitchToState:178(2670ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:34.927150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519222578991996:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:34.901403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519222554164950:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:34.948356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519223433694922:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(174553396ns) {Configure:104(174548935ns) {ProcessConfigured:135(14600603ns) {StartInitialHandshake:157(14596899ns) {DropHandshakes:469(2252ns) {DropIncomingHandshake:441(645ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(1006ns) PrepareNewSessionHandshake:476(5903ns) {SwitchToState:178(2887ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:34.978754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519223680860183:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64624
2025-11-11T17:21:34.974871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519223433694931:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-11T17:21:34.946993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519224994066035:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44859
2025-11-11T17:21:34.979129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519222578991997:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:34.981876Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519223680860183:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:34.997907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519222554164952:2112] [node 4] ICP01 ready to work
2025-11-11T17:21:35.006193Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694916:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.013050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519222578992000:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:21:34.947237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519224994066036:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44859
2025-11-11T17:21:35.014066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519223680860172:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45445655ns) {Configure:104(45439613ns) {ProcessConfigured:135(7408496ns) {StartInitialHandshake:157(7405870ns) {DropHandshakes:469(2556ns) {DropIncomingHandshake:441(610ns) DropOutgoingHandshake:456(429ns)} GenerateSessionVirtualId:988(961ns) PrepareNewSessionHandshake:476(4770ns) {SwitchToState:178(2443ns)}}}}}
2025-11-11T17:21:34.972170Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519224994066035:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:34.977721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519224994066023:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(148328083ns) {Configure:104(148322832ns) {ProcessConfigured:135(16671165ns) {StartInitialHandshake:157(16666380ns) {DropHandshakes:469(3202ns) {DropIncomingHandshake:441(686ns) DropOutgoingHandshake:456(600ns)} GenerateSessionVirtualId:988(1481ns) PrepareNewSessionHandshake:476(5423ns) {SwitchToState:178(3525ns)}}}}}
2025-11-11T17:21:34.989987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519224994066040:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44859
2025-11-11T17:21:34.990054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519224994066041:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44859
2025-11-11T17:21:35.018473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519224403942936:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(81427880ns) {Configure:104(81417218ns) {ProcessConfigured:135(75970ns) {StartInitialHandshake:157(72914ns) {DropHandshakes:469(5075ns) {DropIncomingHandshake:441(924ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(786ns) PrepareNewSessionHandshake:476(2009ns) {SwitchToState:178(560ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.020806Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694931:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.013494Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519222578991997:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.046301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519227975827486:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64624
2025-11-11T17:21:35.024431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519227728662234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-11T17:21:35.036614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132249:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.067715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519222578991996:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(122448731ns) {Configure:104(122443899ns) {ProcessConfigured:135(61220ns) {StartInitialHandshake:157(58995ns) {DropHandshakes:469(3266ns) {DropIncomingHandshake:441(558ns) DropOutgoingHandshake:456(549ns)} GenerateSessionVirtualId:988(656ns) PrepareNewSessionHandshake:476(1552ns) {SwitchToState:178(628ns)}}}}}
2025-11-11T17:21:35.046373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519227975827487:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64624
2025-11-11T17:21:35.036700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132250:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.073156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694928:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.071191Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519222578992000:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.070564Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519224994066036:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.071245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519224994065958:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.046531Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519222554164950:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.073190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519223433694928:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.080224Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519224403943000:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.073693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694921:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.080879Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519224994066041:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.080031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519226873959302:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:21:35.070714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519222554164946:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.080269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519224403943000:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.088207Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519227975827486:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.089963Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519224403943000:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:35.090342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519224403942997:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.090405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:35.090452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519224403942997:2108] [node 6] ICP22 created new session: [5:7571519228698910312:2108]
2025-11-11T17:21:35.090503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519228698910312:2108] [node 6] ICS09 handshake done sender: [5:7571519224403943000:2110] self: [5:7571519224403942999:0] peer: [6:7571519226849132258:0] socket: 77 qp: -1
2025-11-11T17:21:35.090547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519228698910312:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:35.090584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.073706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519223433694921:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.080060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519222578991928:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.081838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519223433694937:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-11T17:21:35.082269Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519223433694928:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:35.088018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519227728662234:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.088745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519223433694921:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:35.071786Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519226849132250:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.088538Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519224994066028:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.088578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519224994066028:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.102937Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519224994066040:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.103917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519229289033340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44859
2025-11-11T17:21:35.104442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519224994065959:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.105124Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519224994066036:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:35.107475Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519224994066028:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:35.071877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519222554164881:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.111411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519224994066023:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.111480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:35.111535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519224994066023:2108] [node 4] ICP22 created new session: [3:7571519229289033343:2108]
2025-11-11T17:21:35.111586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519229289033343:2108] [node 4] ICS09 handshake done sender: [3:7571519224994066028:2111] self: [3:7571519224994066027:0] peer: [4:7571519226873959304:0] socket: 86 qp: -1
2025-11-11T17:21:35.111620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519229289033343:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:35.098314Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519223680860181:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.111659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:35.117170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519224994065958:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.117255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:35.117313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519224994065958:2048] [node 1] ICP22 created new session: [3:7571519229289033345:2048]
2025-11-11T17:21:35.117364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519229289033345:2048] [node 1] ICS09 handshake done sender: [3:7571519224994066036:2116] self: [3:7571519229289033338:0] peer: [1:7571519223433694918:0] socket: 85 qp: -1
2025-11-11T17:21:35.117416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519229289033345:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:35.117471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:35.081328Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519226873959302:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.108616Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694937:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.087643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132260:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.098342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519223680860181:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.106213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519226873959306:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:21:35.089076Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519226849132249:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.101774Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519223680860181:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:35.108749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519223433694923:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.120646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519223680860178:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.108828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:35.089164Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519226849132250:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.133155Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519222578992000:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:35.120688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519223680860178:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.089767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519222554164882:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.147787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:35.134235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519229289033347:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44859
2025-11-11T17:21:35.133836Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.108893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519223433694923:2109] [node 6] ICP22 created new session: [1:7571519227728662240:2109]
2025-11-11T17:21:35.126614Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519227975827487:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.126666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519223680860176:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.149672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:35.148784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519222578991928:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519222578992000:2115] held: no
2025-11-11T17:21:35.108937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519227728662240:2109] [node 6] ICS09 handshake done sender: [1:7571519223433694928:2113] self: [1:7571519223433694927:0] peer: [6:7571519226849132259:0] socket: 102 qp: -1
2025-11-11T17:21:35.126716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:35.126766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519223680860176:2109] [node 6] ICP22 created new session: [2:7571519227975827490:2109]
2025-11-11T17:21:35.126819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519227975827490:2109] [node 6] ICS09 handshake done sender: [2:7571519223680860181:2113] self: [2:7571519223680860180:0] peer: [6:7571519226849132261:0] socket: 92 qp: -1
2025-11-11T17:21:35.126853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519227975827490:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:35.126898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:35.128542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519223680860108:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.136656Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519223680860178:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:35.152045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519223680860109:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.153623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519227975827495:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64624
2025-11-11T17:21:35.154755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:35.154809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519223680860109:2049] [node 3] ICP22 created new session: [2:7571519227975827497:2049]
2025-11-11T17:21:35.154859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519227975827497:2049] [node 3] ICS09 handshake done sender: [2:7571519223680860178:2112] self: [2:7571519223680860177:0] peer: [3:7571519229289033341:0] socket: 91 qp: -1
2025-11-11T17:21:35.154893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519227975827497:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:35.154971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:35.099444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519222554164881:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.156858Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519224994066040:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:35.156234Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519228698910315:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.182286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519222578991928:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.182416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:35.182471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519222578991928:2050] [node 3] ICP22 created new session: [4:7571519226873959309:2050]
2025-11-11T17:21:35.182525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519226873959309:2050] [node 3] ICS09 handshake done sender: [4:7571519222578992000:2115] self: [4:7571519226873959304:0] peer: [3:7571519224994066027:0] socket: 88 qp: -1
2025-11-11T17:21:35.182563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519226873959309:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:35.182635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.108970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519227728662240:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:35.099527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:35.166120Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.109033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:35.156392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519224403942936:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.187020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519223680860182:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.186806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519226873959307:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:21:35.167272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519224994065959:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.099574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519222554164881:2048] [node 1] ICP22 created new session: [6:7571519226849132263:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.187017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519226873959308:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:21:35.200209Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519226873959307:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.200371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519222578991926:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.099627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519226849132263:2048] [node 1] ICS09 handshake done sender: [6:7571519226849132250:2114] self: [6:7571519226849132259:0] peer: [1:7571519223433694927:0] socket: 105 qp: -1
2025-11-11T17:21:35.117412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519223433694857:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.167318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.187065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519223680860182:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.199182Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.213064Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519227975827487:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:35.213349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519223680860108:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519227975827487:2118] held: no
2025-11-11T17:21:35.195298Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519228698910316:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.117532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.167356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519224994065959:2049] [node 2] ICP22 created new session: [3:7571519229289033350:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.222474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519223680860108:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.099662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519226849132263:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:35.201053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519224403942937:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.221764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519222578991999:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.222548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:35.222604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519223680860108:2048] [node 1] ICP22 created new session: [2:7571519227975827501:2048]
2025-11-11T17:21:35.222650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519227975827501:2048] [node 1] ICS09 handshake done sender: [2:7571519227975827487:2118] self: [2:7571519227975827494:0] peer: [1:7571519223433694925:0] socket: 103 qp: -1
2025-11-11T17:21:35.117584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519223433694857:2049] [node 3] ICP22 created new session: [1:7571519227728662242:2049]
2025-11-11T17:21:35.222688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519227975827501:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:35.222737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:35.117654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519227728662242:2049] [node 3] ICS09 handshake done sender: [1:7571519223433694921:2111] self: [1:7571519223433694918:0] peer: [3:7571519229289033338:0] socket: 83 qp: -1
2025-11-11T17:21:35.225120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519223680860109:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(71115920ns) {HandleHandshakeStatus:329(71113347ns) {DropIncomingHandshake:441(24193ns) DropOutgoingHandshake:456(404ns) SwitchToState:178(1002ns) ProcessPendingSessionEvents:495(66084223ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 65149206ns max# 18871138ns}}}
2025-11-11T17:21:35.117691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519227728662242:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:35.118045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:35.120734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519227728662239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-11T17:21:35.167403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519229289033350:2049] [node 2] ICS09 handshake done sender: [3:7571519224994066040:2117] self: [3:7571519229289033341:0] peer: [2:7571519223680860177:0] socket: 95 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.238107Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519223680860182:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:35.239740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519223680860175:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.239798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:35.239845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519223680860175:2108] [node 5] ICP22 created new session: [2:7571519227975827503:2108]
2025-11-11T17:21:35.239919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519227975827503:2108] [node 5] ICS09 handshake done sender: [2:7571519223680860182:2114] self: [2:7571519223680860179:0] peer: [5:7571519228698910318:0] socket: 118 qp: -1
2025-11-11T17:21:35.239959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519227975827503:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:35.239996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:35.221989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910317:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:35.099696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:35.146161Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519227728662239:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.221812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519222578991999:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.241900Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.241845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519227975827499:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64624
2025-11-11T17:21:35.102022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132262:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.167441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519229289033350:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:35.267178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519224403942937:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519228698910316:2117] held: no
2025-11-11T17:21:35.224775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519226873959313:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:21:35.243730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519226873959308:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.244065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519222578991927:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.258461Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519226873959307:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:35.266405Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519222578991999:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:35.146269Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694926:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.167468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:35.245828Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519227975827484:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.104533Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.279801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910320:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.280406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910321:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:35.280923Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519228698910316:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:35.281190Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519228698910315:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:35.167961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519224994065959:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:35.107069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519222554164946:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519222554164950:2111] held: no
2025-11-11T17:21:35.132345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519222554164946:2104] [node 5] ICP35 event processing took too much time PendingConnection:278(24455413ns) {HandleHandshakeStatus:395(24453732ns) {LogHandshakeFail:472(12975ns) {UpdateErrorStateLog:544(5676ns)} DropOutgoingHandshake:456(495ns) ProcessPendingSessionEvents:495(24382005ns) {unordered DropSessionEvent:512 num# 100 sum# 7373102ns max# 4053215ns}}}
2025-11-11T17:21:35.132606Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519222554164950:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:35.149791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519222554164946:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.149842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:35.151168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519222554164946:2104] [node 5] ICP22 created new session: [6:7571519226849132267:2104]
2025-11-11T17:21:35.154062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519226849132267:2104] [node 5] ICS09 handshake done sender: [6:7571519222554164950:2111] self: [6:7571519226849132258:0] peer: [5:7571519224403942999:0] socket: 78 qp: -1
2025-11-11T17:21:35.155949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519226849132267:2104] [node 5] ICS10 traffic start
2025-11-11T17:21:35.161008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:35.189776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519222554164946:2104] [node 5] ICP35 event processing took too much time PendingConnection:278(40211233ns) {HandleHandshakeStatus:329(37872541ns) {DropIncomingHandshake:441(2146ns) DropOutgoingHandshake:456(222ns) SwitchToState:178(394ns) ProcessPendingSessionEvents:495(1911ns)}}
2025-11-11T17:21:35.203253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132269:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.204810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132270:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.206646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132265:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.216299Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519226849132270:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.216483Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519226849132269:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.216673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519222554164952:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.216755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519222554164883:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.224445Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519226849132249:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:35.229325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519222554164882:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.229378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:35.229426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519222554164882:2049] [node 2] ICP22 created new session: [6:7571519226849132275:2049]
2025-11-11T17:21:35.232692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132274:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.242207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519226849132275:2049] [node 2] ICS09 handshake done sender: [6:7571519226849132249:2113] self: [6:7571519226849132261:0] peer: [2:7571519223680860180:0] socket: 104 qp: -1
2025-11-11T17:21:35.253658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519226849132275:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:35.257924Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519226849132269:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:35.266072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:35.290019Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.146291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519223433694926:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.281632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519224403942937:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.245855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519227975827484:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.246841Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519227975827484:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:35.247106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519223680860172:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.247164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:35.247215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519223680860172:2107] [node 4] ICP22 created new session: [2:7571519227975827506:2107]
2025-11-11T17:21:35.256482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519227975827506:2107] [node 4] ICS09 handshake done sender: [2:7571519227975827484:2116] self: [2:7571519227975827483:0] peer: [4:7571519226873959315:0] socket: 126 qp: -1
2025-11-11T17:21:35.258731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519227975827506:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:35.258774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:35.270738Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:35.271843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519223680860176:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:35.282051Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.281686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:35.152367Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519223433694926:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.295043Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.323013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519222578991926:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.323119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:35.323161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519222578991926:2048] [node 1] ICP22 created new session: [4:7571519226873959317:2048]
2025-11-11T17:21:35.323227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519226873959317:2048] [node 1] ICS09 handshake done sender: [4:7571519226873959307:2119] self: [4:7571519226873959312:0] peer: [1:7571519223433694929:0] socket: 94 qp: -1
2025-11-11T17:21:35.323259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519226873959317:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:35.323307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:35.324135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519222578991993:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.324216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:35.324278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519222578991993:2109] [node 6] ICP22 created new session: [4:7571519226873959319:2109]
2025-11-11T17:21:35.324352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519226873959319:2109] [node 6] ICS09 handshake done sender: [4:7571519222578991999:2114] self: [4:7571519222578991998:0] peer: [6:7571519226849132272:0] socket: 89 qp: -1
2025-11-11T17:21:35.324383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519226873959319:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:35.324452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:35.205259Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519224994066032:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.315313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519222554164883:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.283312Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519228698910317:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.324719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519222578991926:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:35.171306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519223433694856:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.205284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519224994066032:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.321790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:35.283398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519224403942936:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.346540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519223680860172:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(96654474ns) {HandleHandshakeStatus:329(96651968ns) {DropIncomingHandshake:441(11073ns) DropOutgoingHandshake:456(507ns) SwitchToState:178(980ns) ProcessPendingSessionEvents:495(69021295ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 63620583ns max# 23412433ns}}}
2025-11-11T17:21:35.347293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519227975827508:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64624
2025-11-11T17:21:35.347351Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:35.285857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519224403942937:2050] [node 3] ICP22 created new session: [5:7571519228698910323:2050]
2025-11-11T17:21:35.171443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:35.209998Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519224994066032:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:35.321856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519222554164883:2050] [node 3] ICP22 created new session: [6:7571519226849132278:2050]
2025-11-11T17:21:35.301950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519228698910323:2050] [node 3] ICS09 handshake done sender: [5:7571519228698910316:2117] self: [5:7571519228698910319:0] peer: [3:7571519224994066031:0] socket: 119 qp: -1
2025-11-11T17:21:35.305805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:35.305874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519224403942936:2049] [node 2] ICP22 created new session: [5:7571519228698910324:2049]
2025-11-11T17:21:35.305927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519228698910324:2049] [node 2] ICS09 handshake done sender: [5:7571519228698910315:2116] self: [5:7571519228698910318:0] peer: [2:7571519223680860179:0] socket: 74 qp: -1
2025-11-11T17:21:35.321917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519226849132278:2050] [node 3] ICS09 handshake done sender: [6:7571519226849132269:2121] self: [6:7571519226849132273:0] peer: [3:7571519224994066033:0] socket: 125 qp: -1
2025-11-11T17:21:35.321957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519226849132278:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:35.321997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:35.322745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519226849132276:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26975
2025-11-11T17:21:35.326048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519222554164882:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(95876672ns) {HandleHandshakeStatus:329(95874200ns) {DropIncomingHandshake:441(826ns) DropOutgoingHandshake:456(4037ns) SwitchToState:178(1046ns) ProcessPendingSessionEvents:495(39463069ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39385067ns max# 12010808ns}}}
2025-11-11T17:21:35.326169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519222554164883:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:35.328351Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519226849132270:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:35.210831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519224994066025:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.171491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519223433694856:2048] [node 2] ICP22 created new session: [1:7571519227728662250:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.342615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519222554164952:2112] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.305970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519228698910324:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.210907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:35.342708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:35.342764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519222554164952:2112] [node 4] ICP22 created new session: [6:7571519226849132280:2112]
2025-11-11T17:21:35.342831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519226849132280:2112] [node 4] ICS09 handshake done sender: [6:7571519226849132270:2122] self: [6:7571519226849132272:0] peer: [4:7571519222578991998:0] socket: 127 qp: -1
2025-11-11T17:21:35.342909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519226849132280:2112] [node 4] ICS10 traffic start
2025-11-11T17:21:35.342962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:35.171541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519227728662250:2048] [node 2] ICS09 handshake done sender: [1:7571519223433694926:2112] self: [1:7571519223433694925:0] peer: [2:7571519227975827494:0] socket: 101 qp: -1
2025-11-11T17:21:35.210958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519224994066025:2109] [node 5] ICP22 created new session: [3:7571519229289033354:2109]
2025-11-11T17:21:35.211010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519229289033354:2109] [node 5] ICS09 handshake done sender: [3:7571519224994066032:2113] self: [3:7571519224994066031:0] peer: [5:7571519228698910319:0] socket: 81 qp: -1
2025-11-11T17:21:35.211046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519229289033354:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:35.211088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:35.217384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519229289033352:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44859
2025-11-11T17:21:35.217854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519224994066034:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.217878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519224994066034:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.219137Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519224994066034:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:35.231068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519224994066026:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.231107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:35.231156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519224994066026:2110] [node 6] ICP22 created new session: [3:7571519229289033357:2110]
2025-11-11T17:21:35.231202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519229289033357:2110] [node 6] ICS09 handshake done sender: [3:7571519224994066034:2114] self: [3:7571519224994066033:0] peer: [6:7571519226849132273:0] socket: 97 qp: -1
2025-11-11T17:21:35.231236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519229289033357:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:35.231272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:35.248251Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519229289033352:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.248409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519224994066026:2110] [node 6] ICP18 handshake (actor [3:7571519229289033352:2121]) is too old
2025-11-11T17:21:35.306015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:35.355335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519226873959316:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42250
2025-11-11T17:21:35.171577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519227728662250:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:35.307512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519224403942936:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(23317236ns) {HandleHandshakeStatus:329(23315226ns) {DropIncomingHandshake:441(763ns) DropOutgoingHandshake:456(6132ns) SwitchToState:178(788ns) ProcessPendingSessionEvents:495(495854ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 446279ns max# 50567ns}}}
2025-11-11T17:21:35.171619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:35.308047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519228698910323:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:35.388095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519222578991993:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(63155632ns) {HandleHandshakeStatus:329(63153398ns) {DropIncomingHandshake:441(13995ns) DropOutgoingHandshake:456(380ns) SwitchToState:178(246ns) ProcessPendingSessionEvents:495(62724550ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 61277657ns max# 12552333ns}}}
2025-11-11T17:21:35.206069Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694930:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.389748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519222578991927:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519226873959308:2120] held: no
2025-11-11T17:21:35.386063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519222554164952:2112] [node 4] ICP35 event processing took too much time PendingConnection:278(42623478ns) {HandleHandshakeStatus:329(42618770ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(7273ns) SwitchToState:178(544ns) ProcessPendingSessionEvents:495(30682041ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28606535ns max# 12468303ns}}}
2025-11-11T17:21:35.314113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:35.206089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519223433694930:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.229304Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519223433694930:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:35.254302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519223433694922:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.254431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:35.254502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519223433694922:2108] [node 4] ICP22 created new session: [1:7571519227728662253:2108]
2025-11-11T17:21:35.254561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519227728662253:2108] [node 4] ICS09 handshake done sender: [1:7571519223433694930:2114] self: [1:7571519223433694929:0] peer: [4:7571519226873959312:0] socket: 120 qp: -1
2025-11-11T17:21:35.254604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519227728662253:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:35.254650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:35.272286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519223433694857:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(151103416ns) {HandleHandshakeStatus:329(151101250ns) {DropIncomingHandshake:441(27514ns) DropOutgoingHandshake:456(579ns) SwitchToState:178(664ns) ProcessPendingSessionEvents:495(133308892ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 133158560ns max# 13359440ns}}}
2025-11-11T17:21:35.355271Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519223433694933:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.355302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519223433694933:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:35.356461Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519223433694933:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:35.380267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519223433694924:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.380330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:35.380384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519223433694924:2110] [node 5] ICP22 created new session: [1:7571519227728662256:2110]
2025-11-11T17:21:35.380433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519227728662256:2110] [node 5] ICS09 handshake done sender: [1:7571519223433694933:2116] self: [1:7571519223433694932:0] peer: [5:7571519228698910327:0] socket: 121 qp: -1
2025-11-11T17:21:35.380471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519227728662256:2110] [node 5] ICS10 traffic start
2025-11-11T17:21:35.380548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:35.400629Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519226873959308:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:35.329191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519224403942937:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(43246639ns) {HandleHandshakeStatus:329(43230642ns) {DropIncomingHandshake:441(1026ns) DropOutgoingHandshake:456(480ns) SwitchToState:178(643ns) ProcessPendingSessionEvents:495(1235ns)}}
2025-11-11T17:21:35.350919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519224403942935:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.377044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910328:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:35.383373Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519228698910317:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:35.392512Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.397484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519224403942935:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.397963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:35.400713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519224403942935:2048] [node 1] ICP22 created new session: [5:7571519228698910329:2048]
2025-11-11T17:21:35.400765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519228698910329:2048] [node 1] ICS09 handshake done sender: [5:7571519228698910317:2118] self: [5:7571519228698910327:0] peer: [1:7571519223433694932:0] socket: 122 qp: -1
2025-11-11T17:21:35.400804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519228698910329:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:35.400855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:35.402878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519224403942935:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:35.403897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519222578991927:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.405035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:35.405093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519222578991927:2049] [node 2] ICP22 created new session: [4:7571519226873959322:2049]
2025-11-11T17:21:35.405134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519226873959322:2049] [node 2] ICS09 handshake done sender: [4:7571519226873959308:2120] self: [4:7571519226873959315:0] peer: [2:7571519227975827483:0] socket: 87 qp: -1
2025-11-11T17:21:35.405170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519226873959322:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:35.405807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.408843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910331:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:35.409533Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519228698910331:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.409730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519224403943004:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:35.409948Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519226873959301:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:35.409979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519226873959301:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:35.410941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519228698910333:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53284
2025-11-11T17:21:35.411193Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519226873959301:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:35.411639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519222578991996:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:35.411840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:35.411907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519222578991996:2110] [node 5] ICP22 created new session: [4:7571519226873959325:2110]
2025-11-11T17:21:35.411957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519226873959325:2110] [node 5] ICS09 handshake done sender: [4:7571519226873959301:2116] self: [4:7571519226873959300:0] peer: [5:7571519228698910332:0] socket: 93 qp: -1
2025-11-11T17:21:35.411990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519226873959325:2110] [node 5] ICS10 traffic start
2025-11-11T17:21:35.412024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:35.413450Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519228698910331:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:35.413666Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:35.414712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519224403943004:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519228698910331:2122] held: no
2025-11-11T17:21:35.415323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519224403943004:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:35.415448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:35.415528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519224403943004:2111] [node 4] ICP22 created new session: [5:7571519228698910334:2111]
2025-11-11T17:21:35.415569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519228698910334:2111] [node 4] ICS09 handshake done sender: [5:7571519228698910331:2122] self: [5:7571519228698910332:0] peer: [4:7571519226873959300:0] socket: 96 qp: -1
2025-11-11T17:21:35.415611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519228698910334:2111] [node 4] ICS10 traffic start
2025-11-11T17:21:35.415670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:35.475991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:35.476335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519227975827501:2048] [node 1] ICS01 socket: 103 reason# EndOfStream
2025-11-11T17:21:35.476392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519223680860108:2048] [node 1] ICP30 unregister session Session# [2:7571519227975827501:2048] VirtualId# [2:7571519227975827494:0]
2025-11-11T17:21:35.476432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519227975827501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.476551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:35.476610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:35.478445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:35.478896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519229289033345:2048] [node 1] ICS01 socket: 85 reason# EndOfStream
2025-11-11T17:21:35.478963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519224994065958:2048] [node 1] ICP30 unregister session Session# [3:7571519229289033345:2048] VirtualId# [3:7571519229289033338:0]
2025-11-11T17:21:35.478999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519229289033345:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.479100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:35.479147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:35.481760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:35.481981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519226873959317:2048] [node 1] ICS07 socket disconnect 94 reason# EndOfStream
2025-11-11T17:21:35.482010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519226873959317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.482085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:35.482121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519226873959317:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:35.482943Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address
2025-11-11T17:21:35.483242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519222578991926:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:35.483287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519226873959317:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:35.483318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519222578991926:2048] [node 1] ICP30 unregister session Session# [4:7571519226873959317:2048] VirtualId# [4:7571519226873959312:0]
2025-11-11T17:21:35.483415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:35.483487Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519222578991926:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.483502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.489058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:35.489276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519228698910329:2048] [node 1] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-11T17:21:35.489309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519228698910329:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.484716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:35.484932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519226849132263:2048] [node 1] ICS07 socket disconnect 105 reason# EndOfStream
2025-11-11T17:21:35.484959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519226849132263:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.485025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:35.485059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519226849132263:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:35.489396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:35.489431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519228698910329:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:35.490636Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address
2025-11-11T17:21:35.494169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519224403942935:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:35.507474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519222578991926:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:35.490628Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address
2025-11-11T17:21:35.501793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519222554164881:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:35.507848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519226849132263:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:35.507902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519222554164881:2048] [node 1] ICP30 unregister session Session# [6:7571519226849132263:2048] VirtualId# [6:7571519226849132259:0]
2025-11-11T17:21:35.507966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:35.508010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519222554164881:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.508025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.508320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [6:7571519222554164881:2048] [node 1] ICP35 event processing took too much time StateWork:293(17246193ns) {HandleHandshakeStatus:395(13829240ns) {LogHandshakeFail:472(7500610ns) {UpdateErrorStateLog:544(6903ns)} DropOutgoingHandshake:456(773ns) ProcessPendingSessionEvents:495(512ns) UnregisterSession:538(52646ns) {ProcessPendingSessionEvents:495(481ns) SwitchToInitialState:201(1437ns) {SwitchToState:178(625ns)}} TransitToErrorState:752(58211ns) {SwitchToState:178(5043ns) ProcessPendingSessionEvents:495(304ns) DropHandshakes:469(3510ns) {DropIncomingHandshake:441(980ns) DropOutgoingHandshake:456(547ns)}}}}
2025-11-11T17:21:35.507401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519228698910329:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:35.508403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519224403942935:2048] [node 1] ICP30 unregister session Session# [5:7571519228698910329:2048] VirtualId# [5:7571519228698910327:0]
2025-11-11T17:21:35.508438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:35.508470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519224403942935:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.508487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.508704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [5:7571519224403942935:2048] [node 1] ICP35 event processing took too much time StateWork:293(17719696ns) {HandleHandshakeStatus:395(17717164ns) {LogHandshakeFail:472(7586ns) {UpdateErrorStateLog:544(1614ns)} DropOutgoingHandshake:456(2733ns) ProcessPendingSessionEvents:495(511ns) UnregisterSession:538(43901ns) {ProcessPendingSessionEvents:495(50ns) SwitchToInitialState:201(450ns) {SwitchToState:178(179ns)}} TransitToErrorState:752(45004ns) {SwitchToState:178(4322ns) ProcessPendingSessionEvents:495(48ns) DropHandshakes:469(765ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(199ns)}}}}
2025-11-11T17:21:35.518778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519224403942935:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:35.529910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519222554164881:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:35.537399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:35.537724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519229289033350:2049] [node 2] ICS01 socket: 95 reason# EndOfStream
2025-11-11T17:21:35.537773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519224994065959:2049] [node 2] ICP30 unregister session Session# [3:7571519229289033350:2049] VirtualId# [3:7571519229289033341:0]
2025-11-11T17:21:35.537809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519229289033350:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.537916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:35.537964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:35.545717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:35.545722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:35.546019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519226873959322:2049] [node 2] ICS01 socket: 87 reason# EndOfStream
2025-11-11T17:21:35.546025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519226849132275:2049] [node 2] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-11T17:21:35.546073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519226849132275:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.546193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:35.546267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519226849132275:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:35.546087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519222578991927:2049] [node 2] ICP30 unregister session Session# [4:7571519226873959322:2049] VirtualId# [4:7571519226873959315:0]
2025-11-11T17:21:35.546135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519226873959322:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.546242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:35.546307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:35.541771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:35.542030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519228698910324:2049] [node 2] ICS07 socket disconnect 74 reason# EndOfStream
2025-11-11T17:21:35.542062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519228698910324:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.542183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:35.542230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519228698910324:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:35.546797Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address
2025-11-11T17:21:35.547087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519224403942936:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:35.547144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519228698910324:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:35.547182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519224403942936:2049] [node 2] ICP30 unregister session Session# [5:7571519228698910324:2049] VirtualId# [5:7571519228698910318:0]
2025-11-11T17:21:35.547277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:35.547306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519224403942936:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.547331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.557954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519224403942936:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:35.558200Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address
2025-11-11T17:21:35.558516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519222554164882:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:35.558566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519226849132275:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:35.558615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519222554164882:2049] [node 2] ICP30 unregister session Session# [6:7571519226849132275:2049] VirtualId# [6:7571519226849132261:0]
2025-11-11T17:21:35.558709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:35.558757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519222554164882:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.558800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.569740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519222554164882:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:35.634999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:35.635201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519226873959309:2050] [node 3] ICS01 socket: 88 reason# EndOfStream
2025-11-11T17:21:35.635240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519222578991928:2050] [node 3] ICP30 unregister session Session# [4:7571519226873959309:2050] VirtualId# [4:7571519226873959304:0]
2025-11-11T17:21:35.635275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519226873959309:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.635369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:35.635421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:35.638946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:35.639243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519228698910323:2050] [node 3] ICS01 socket: 119 reason# EndOfStream
2025-11-11T17:21:35.640661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519224403942937:2050] [node 3] ICP30 unregister session Session# [5:7571519228698910323:2050] VirtualId# [5:7571519228698910319:0]
2025-11-11T17:21:35.640702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519228698910323:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.640833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:35.640884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:35.642032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:35.642333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519226849132278:2050] [node 3] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-11T17:21:35.642383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519226849132278:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.642508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:35.642601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519226849132278:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:35.643678Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Couldn't connect to any resolved address
2025-11-11T17:21:35.645658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519222554164883:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:35.651865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519226849132278:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:35.651915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519222554164883:2050] [node 3] ICP30 unregister session Session# [6:7571519226849132278:2050] VirtualId# [6:7571519226849132273:0]
2025-11-11T17:21:35.651985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:35.654743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519222554164883:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.654770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:35.665791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519222554164883:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:35.675630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:35.675874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519226849132280:2112] [node 4] ICS01 socket: 127 reason# EndOfStream
2025-11-11T17:21:35.675930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519222554164952:2112] [node 4] ICP30 unregister session Session# [6:7571519226849132280:2112] VirtualId# [6:7571519226849132272:0]
2025-11-11T17:21:35.675972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519226849132280:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.676079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:35.676123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:35.677728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:35.677924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519228698910334:2111] [node 4] ICS01 socket: 96 reason# EndOfStream
2025-11-11T17:21:35.677972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519224403943004:2111] [node 4] ICP30 unregister session Session# [5:7571519228698910334:2111] VirtualId# [5:7571519228698910332:0]
2025-11-11T17:21:35.678005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519228698910334:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.678087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:35.678129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:35.733919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:35.734126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519226849132267:2104] [node 5] ICS01 socket: 78 reason# EndOfStream
2025-11-11T17:21:35.734173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519222554164946:2104] [node 5] ICP30 unregister session Session# [6:7571519226849132267:2104] VirtualId# [6:7571519226849132258:0]
2025-11-11T17:21:35.734206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519226849132267:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:35.734311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:35.734354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:35.800254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519229908938167:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:35.800689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519229908938168:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:35.819914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519230633544026:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:35.819967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519230633544027:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:35.840165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519228986950180:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:35.840190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519228986950181:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:35.862373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519227133655388:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:35.862420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519227133655390:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:35.863059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519227133655389:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:35.885125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519227432127130:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:35.885164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519227432127131:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:35.885478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519227432127129:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:35.909056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519229319325253:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:35.909096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519229319325254:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:35.909186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519229319325255:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:36.126244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519233614292609:2103] [node 5] ICP01 ready to work
2025-11-11T17:21:36.126386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519233614292610:2106] [node 4] ICP01 ready to work
2025-11-11T17:21:36.153767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519234203905524:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:36.201657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519234203905535:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:36.191906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519234928511385:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:36.231366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519234928511387:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:36.214324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519233281917547:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:36.235417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519234203905537:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:36.235442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519234928511386:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:36.319250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519233281917556:2118] [node 6] ICP01 ready to work
2025-11-11T17:21:36.342826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519231727094489:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:36.349705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519231428622747:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:36.374110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519233614292610:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(219992816ns) {Configure:104(219982806ns) {ProcessConfigured:135(69292ns) {StartInitialHandshake:157(66972ns) {DropHandshakes:469(2817ns) {DropIncomingHandshake:441(1059ns) DropOutgoingHandshake:456(443ns)} GenerateSessionVirtualId:988(948ns) PrepareNewSessionHandshake:476(1644ns) {SwitchToState:178(478ns)}}}}}
2025-11-11T17:21:36.366780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519231727094490:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:36.368344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519233281917557:2114] [node 5] ICP01 ready to work
2025-11-11T17:21:36.368199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519231428622748:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.383884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519229319325255:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(234592160ns) {Configure:104(234585672ns) {ProcessConfigured:135(16875136ns) {StartInitialHandshake:157(16872134ns) {DropHandshakes:469(3046ns) {DropIncomingHandshake:441(573ns) DropOutgoingHandshake:456(661ns)} GenerateSessionVirtualId:988(1164ns) PrepareNewSessionHandshake:476(2255ns) {SwitchToState:178(837ns)}}}}}
2025-11-11T17:21:36.410050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519233614292609:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(265071713ns) {Configure:104(265062661ns) {ProcessConfigured:135(145700ns) {StartInitialHandshake:157(142045ns) {DropHandshakes:469(1993ns) {DropIncomingHandshake:441(433ns) DropOutgoingHandshake:456(468ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:476(1623ns) {SwitchToState:178(599ns)}}}}}
2025-11-11T17:21:36.407968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519234203905538:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43047
2025-11-11T17:21:36.399484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519230633544026:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21180903ns) {Configure:104(21173953ns) {ProcessConfigured:135(82549ns) {StartInitialHandshake:157(80024ns) {DropHandshakes:469(2642ns) {DropIncomingHandshake:441(540ns) DropOutgoingHandshake:456(547ns)} GenerateSessionVirtualId:988(1081ns) PrepareNewSessionHandshake:476(2120ns) {SwitchToState:178(664ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.423615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519233281917547:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(193848626ns) {Configure:104(193843977ns) {ProcessConfigured:135(21969796ns) {StartInitialHandshake:157(21966060ns) {DropHandshakes:469(2644ns) {DropIncomingHandshake:441(551ns) DropOutgoingHandshake:456(488ns)} GenerateSessionVirtualId:988(976ns) PrepareNewSessionHandshake:476(5323ns) {SwitchToState:178(2405ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.425762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519229908938168:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(284828434ns) {Configure:104(284817824ns) {ProcessConfigured:135(4914193ns) {StartInitialHandshake:157(4911461ns) {DropHandshakes:469(2375901ns) {DropIncomingHandshake:441(2642ns) DropOutgoingHandshake:456(342ns)} GenerateSessionVirtualId:988(3722ns) PrepareNewSessionHandshake:476(2882ns) {SwitchToState:178(720ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.440262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519227133655388:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(56288042ns) {Configure:104(56282938ns) {ProcessConfigured:135(44251ns) {StartInitialHandshake:157(42200ns) {DropHandshakes:469(1908ns) {DropIncomingHandshake:441(323ns) DropOutgoingHandshake:456(603ns)} GenerateSessionVirtualId:988(651ns) PrepareNewSessionHandshake:476(1172ns) {SwitchToState:178(528ns)}}}}}
2025-11-11T17:21:36.445566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622749:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
2025-11-11T17:21:36.438853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519234928511390:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29939
2025-11-11T17:21:36.434925Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905538:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.436434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519233281917556:2118] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(50169071ns) {Configure:104(50165247ns) {ProcessConfigured:135(18313066ns) {StartInitialHandshake:157(18310120ns) {DropHandshakes:469(2288ns) {DropIncomingHandshake:441(472ns) DropOutgoingHandshake:456(359ns)} GenerateSessionVirtualId:988(964ns) PrepareNewSessionHandshake:476(8107ns) {SwitchToState:178(4206ns)}}}}}
2025-11-11T17:21:36.446710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292627:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.457159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519231428622749:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.479904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292628:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.469418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519230633544027:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(57857536ns) {Configure:104(57852728ns) {ProcessConfigured:135(4127324ns) {StartInitialHandshake:157(4123920ns) {DropHandshakes:469(2718ns) {DropIncomingHandshake:441(591ns) DropOutgoingHandshake:456(497ns)} GenerateSessionVirtualId:988(1414ns) PrepareNewSessionHandshake:476(2981ns) {SwitchToState:178(1449ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.501348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519234203905554:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43047
2025-11-11T17:21:36.473755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622758:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
2025-11-11T17:21:36.484145Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519233614292628:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.513958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519228986950181:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(147702713ns) {Configure:104(140650570ns) {ProcessConfigured:135(32440627ns) {StartInitialHandshake:157(32437094ns) {DropHandshakes:469(1980ns) {DropIncomingHandshake:441(602ns) DropOutgoingHandshake:456(190ns)} GenerateSessionVirtualId:988(766ns) PrepareNewSessionHandshake:476(4226ns) {SwitchToState:178(2040ns)}}}}}
2025-11-11T17:21:36.469231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519227432127129:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34194606ns) {Configure:104(34184713ns) {ProcessConfigured:135(62918ns) {StartInitialHandshake:157(59022ns) {DropHandshakes:469(2761ns) {DropIncomingHandshake:441(559ns) DropOutgoingHandshake:456(550ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:476(1870ns) {SwitchToState:178(546ns)}}}}}
2025-11-11T17:21:36.471537Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511390:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.484326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519229319325254:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.477957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519234928511386:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(46630494ns) {Configure:104(46625621ns) {ProcessConfigured:135(12805479ns) {StartInitialHandshake:157(12801946ns) {DropHandshakes:469(3054ns) {DropIncomingHandshake:441(638ns) DropOutgoingHandshake:456(521ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:476(3802ns) {SwitchToState:178(1069ns)}}}}}
2025-11-11T17:21:36.515904Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905554:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.515359Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519231727094490:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(53115417ns) {Configure:104(53110061ns) {ProcessConfigured:135(67324ns) {StartInitialHandshake:157(64502ns) {DropHandshakes:469(3677ns) {DropIncomingHandshake:441(649ns) DropOutgoingHandshake:456(702ns)} GenerateSessionVirtualId:988(974ns) PrepareNewSessionHandshake:476(1983ns) {SwitchToState:178(775ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.509218Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519233614292627:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.478119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519234928511393:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29939
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.540238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622762:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
2025-11-11T17:21:36.527434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519229319325253:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.529221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519234203905553:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43047
2025-11-11T17:21:36.501355Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511393:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.535899Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905540:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.534943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519229319325253:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18077433ns) {IncomingHandshake:261(18074308ns) {DropIncomingHandshake:441(648ns) PrepareNewSessionHandshake:476(2691ns) {SwitchToState:178(1107ns)} GenerateSessionVirtualId:988(912ns) IssueIncomingHandshakeReply:193(9743254ns) {SwitchToState:178(1462ns)}}}
2025-11-11T17:21:36.503685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511392:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.538559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292632:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.547904Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519231428622758:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.550977Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519231428622762:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.535937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519234203905540:2120] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.503734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519234928511392:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.515960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519230633544026:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.534161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519234928511392:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:36.549775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519234928511387:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.549855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:36.550573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519234928511387:2109] [node 6] ICP22 created new session: [2:7571519234928511407:2109]
2025-11-11T17:21:36.551170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519234928511404:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29939
2025-11-11T17:21:36.536041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905544:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.555865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519234928511407:2109] [node 6] ICS09 handshake done sender: [2:7571519234928511392:2112] self: [2:7571519234928511391:0] peer: [6:7571519233614292629:0] socket: 96 qp: -1
2025-11-11T17:21:36.555928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519234928511407:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:36.540876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292631:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.536066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519234203905544:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.537898Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905553:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.538861Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519234203905544:2122] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:36.550689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519234203905537:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.550747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:36.550788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519234203905537:2109] [node 6] ICP22 created new session: [1:7571519234203905558:2109]
2025-11-11T17:21:36.555982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905542:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.556010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519234203905542:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.554527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519227133655388:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.557076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519227432127131:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(84586283ns) {Configure:104(84580009ns) {ProcessConfigured:135(4101362ns) {StartInitialHandshake:157(4097852ns) {DropHandshakes:469(1717ns) {DropIncomingHandshake:441(395ns) DropOutgoingHandshake:456(356ns)} GenerateSessionVirtualId:988(1272ns) PrepareNewSessionHandshake:476(3006ns) {SwitchToState:178(983ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.568153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519234203905558:2109] [node 6] ICS09 handshake done sender: [1:7571519234203905544:2122] self: [1:7571519234203905543:0] peer: [6:7571519233614292630:0] socket: 87 qp: -1
2025-11-11T17:21:36.569795Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519234203905540:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:36.547591Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519233614292627:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:36.563516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:36.554883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519227133655389:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.569149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519227432127130:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(94102355ns) {Configure:104(94096469ns) {ProcessConfigured:135(2250336ns) {StartInitialHandshake:157(2246731ns) {DropHandshakes:469(3775ns) {DropIncomingHandshake:441(933ns) DropOutgoingHandshake:456(1053ns)} GenerateSessionVirtualId:988(1916ns) PrepareNewSessionHandshake:476(2320ns) {SwitchToState:178(1045ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.549627Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519233614292628:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:36.563564Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511404:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.574031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519234203905558:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:36.574142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519231428622747:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(92771390ns) {Configure:104(92763671ns) {ProcessConfigured:135(4365265ns) {StartInitialHandshake:157(4363070ns) {DropHandshakes:469(1543ns) {DropIncomingHandshake:441(355ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:476(2617ns) {SwitchToState:178(312ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.553748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519229319325253:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.587067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519228986950180:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(50851649ns) {Configure:104(50844787ns) {ProcessConfigured:135(73390ns) {StartInitialHandshake:157(70328ns) {DropHandshakes:469(4614ns) {DropIncomingHandshake:441(1324ns) DropOutgoingHandshake:456(682ns)} GenerateSessionVirtualId:988(1248ns) PrepareNewSessionHandshake:476(2713ns) {SwitchToState:178(1122ns)}}}}}
2025-11-11T17:21:36.587717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519233281917564:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12591
2025-11-11T17:21:36.588539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519233281917568:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12591
2025-11-11T17:21:36.588596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519233281917569:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12591
2025-11-11T17:21:36.603253Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917564:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.618215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519231727094489:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(82522714ns) {Configure:104(82512363ns) {ProcessConfigured:135(12275345ns) {StartInitialHandshake:157(12271839ns) {DropHandshakes:469(2308ns) {DropIncomingHandshake:441(610ns) DropOutgoingHandshake:456(569ns)} GenerateSessionVirtualId:988(2788ns) PrepareNewSessionHandshake:476(3769ns) {SwitchToState:178(1890ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.625748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094502:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.564038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511395:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.578895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:36.554546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.578745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622767:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.564063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519234928511395:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.580098Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519234203905542:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:36.554625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519229319325253:2048] [node 1] ICP22 created new session: [6:7571519233614292633:2048]
2025-11-11T17:21:36.602293Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519231428622762:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:36.576163Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519234928511395:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.658580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094505:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.653177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519233281917571:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12591
2025-11-11T17:21:36.596559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519229908938167:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.658781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094506:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.554672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519233614292633:2048] [node 1] ICS09 handshake done sender: [6:7571519233614292627:2115] self: [6:7571519233614292630:0] peer: [1:7571519234203905543:0] socket: 88 qp: -1
2025-11-11T17:21:36.609776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519227133655389:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.601071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519234928511410:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29939
2025-11-11T17:21:36.638335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622770:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
2025-11-11T17:21:36.643386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622769:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
2025-11-11T17:21:36.643385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:36.646850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519227133655389:2049] [node 2] ICP22 created new session: [4:7571519231428622771:2049]
2025-11-11T17:21:36.649322Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:36.653859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519231428622771:2049] [node 2] ICS09 handshake done sender: [4:7571519231428622762:2116] self: [4:7571519231428622766:0] peer: [2:7571519234928511394:0] socket: 103 qp: -1
2025-11-11T17:21:36.655836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519231428622771:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:36.655896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:36.659174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519227133655388:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519231428622758:2115] held: no
2025-11-11T17:21:36.659858Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519231428622770:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.660152Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519231428622758:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:36.667595Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519231727094505:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.554726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519233614292633:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:36.596622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:36.663330Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917568:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.619881Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:36.653632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519230633544026:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519234928511393:2113] held: no
2025-11-11T17:21:36.665845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519234928511385:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.665930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:36.665983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519234928511385:2107] [node 4] ICP22 created new session: [2:7571519234928511414:2107]
2025-11-11T17:21:36.666035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519234928511414:2107] [node 4] ICS09 handshake done sender: [2:7571519234928511395:2114] self: [2:7571519234928511394:0] peer: [4:7571519231428622766:0] socket: 97 qp: -1
2025-11-11T17:21:36.666078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519234928511414:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:36.666153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:36.671418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519230633544026:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22348046ns) {HandleHandshakeStatus:395(22344526ns) {LogHandshakeFail:472(16154ns) {UpdateErrorStateLog:544(5885ns)} DropOutgoingHandshake:456(1343ns) ProcessPendingSessionEvents:495(16909628ns) {unordered DropSessionEvent:512 num# 100 sum# 16874675ns max# 7579445ns}}}
2025-11-11T17:21:36.687321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519234928511393:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.687473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519227133655388:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.695988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519234928511385:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(29429314ns) {HandleHandshakeStatus:329(29425919ns) {DropIncomingHandshake:441(17339ns) DropOutgoingHandshake:456(428ns) SwitchToState:178(1002ns) ProcessPendingSessionEvents:495(11675630ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11614004ns max# 10982348ns}}}
2025-11-11T17:21:36.671922Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519231727094494:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.554776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:36.666590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519233281917557:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(125096694ns) {Configure:104(125090280ns) {ProcessConfigured:135(13178439ns) {StartInitialHandshake:157(13174917ns) {DropHandshakes:469(369348ns) {DropIncomingHandshake:441(868ns) DropOutgoingHandshake:456(1057ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:476(3439ns) {SwitchToState:178(1753ns)}}}}}
2025-11-11T17:21:36.596664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519229908938167:2048] [node 2] ICP22 created new session: [1:7571519234203905564:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.671977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519231727094494:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.557457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519229319325254:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.687545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:36.557512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:36.557563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519229319325254:2049] [node 2] ICP22 created new session: [6:7571519233614292635:2049]
2025-11-11T17:21:36.557623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519233614292635:2049] [node 2] ICS09 handshake done sender: [6:7571519233614292628:2116] self: [6:7571519233614292629:0] peer: [2:7571519234928511391:0] socket: 98 qp: -1
2025-11-11T17:21:36.557661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519233614292635:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:36.557695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:36.559008Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:36.560749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519229319325254:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:36.595258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519229319325253:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(40728761ns) {HandleHandshakeStatus:329(40724987ns) {DropIncomingHandshake:441(1135ns) DropOutgoingHandshake:456(7073ns) SwitchToState:178(2820ns) ProcessPendingSessionEvents:495(25838589ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25779474ns max# 17520622ns}}}
2025-11-11T17:21:36.636744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292639:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.636902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292640:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.651575Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519233614292639:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.653405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519233614292609:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.697697Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519233614292640:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.697940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519229319325255:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.698524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292642:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.740332Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519233614292639:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:36.666867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519228986950180:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.734472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519230633544026:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.690082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519227133655388:2048] [node 1] ICP22 created new session: [4:7571519231428622775:2048]
2025-11-11T17:21:36.688760Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519231727094502:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.596732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519234203905564:2048] [node 2] ICS09 handshake done sender: [1:7571519234203905540:2120] self: [1:7571519234203905539:0] peer: [2:7571519234928511405:0] socket: 84 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.669926Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917569:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.767782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519233614292609:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.698117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519227133655390:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.734551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:36.696698Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519231727094494:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:36.596770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519234203905564:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.767878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:36.689205Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917571:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.746024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519230633544026:2048] [node 1] ICP22 created new session: [2:7571519234928511416:2048]
2025-11-11T17:21:36.698531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519231428622775:2048] [node 1] ICS09 handshake done sender: [4:7571519231428622758:2115] self: [4:7571519231428622764:0] peer: [1:7571519234203905541:0] socket: 89 qp: -1
2025-11-11T17:21:36.713703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519227432127130:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.702018Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917559:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.596811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:36.767931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519233614292609:2103] [node 5] ICP22 created new session: [6:7571519233614292645:2103]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.758535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519234928511413:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29939
2025-11-11T17:21:36.757984Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519231727094506:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.758772Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511400:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.758795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519234928511400:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.760270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519234928511416:2048] [node 1] ICS09 handshake done sender: [2:7571519234928511393:2113] self: [2:7571519234928511405:0] peer: [1:7571519234203905539:0] socket: 86 qp: -1
2025-11-11T17:21:36.768894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519234928511416:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:36.785740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:36.788526Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519234928511400:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:36.820173Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511413:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.820230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519234928511386:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.824217Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519230633544026:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(61384467ns) {HandleHandshakeStatus:329(61380835ns) {DropIncomingHandshake:441(751ns) DropOutgoingHandshake:456(600ns) SwitchToState:178(597ns) ProcessPendingSessionEvents:495(1917ns)}}
2025-11-11T17:21:36.773911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519233614292645:2103] [node 5] ICS09 handshake done sender: [6:7571519233614292639:2119] self: [6:7571519233614292641:0] peer: [5:7571519231727094493:0] socket: 121 qp: -1
2025-11-11T17:21:36.702042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519233281917559:2126] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.698590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519231428622775:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:36.597825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519234203905524:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.713760Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519233281917559:2126] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:36.758128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519227432127129:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.698650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:36.597870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.826811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:36.778063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519233614292645:2103] [node 5] ICS10 traffic start
2025-11-11T17:21:36.778121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:36.760976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094511:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.826901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519234928511386:2108] [node 5] ICP22 created new session: [2:7571519234928511420:2108]
2025-11-11T17:21:36.597916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519234203905524:2107] [node 4] ICP22 created new session: [1:7571519234203905566:2107]
2025-11-11T17:21:36.733767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917561:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.891090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292644:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.733409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622778:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
2025-11-11T17:21:36.761139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.768736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519231727094490:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.778216Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519231727094511:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.775007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:36.778523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519231727094490:2109] [node 6] ICP22 created new session: [5:7571519231727094513:2109]
2025-11-11T17:21:36.795634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519231727094513:2109] [node 6] ICS09 handshake done sender: [5:7571519231727094494:2112] self: [5:7571519231727094493:0] peer: [6:7571519233614292641:0] socket: 112 qp: -1
2025-11-11T17:21:36.798140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519231727094513:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:36.798184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:36.808902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519231727094489:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.809117Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519231727094502:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:36.810739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519227432127130:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.810866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:36.810921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519227432127130:2049] [node 2] ICP22 created new session: [5:7571519231727094518:2049]
2025-11-11T17:21:36.810969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519231727094518:2049] [node 2] ICS09 handshake done sender: [5:7571519231727094502:2115] self: [5:7571519231727094509:0] peer: [2:7571519234928511398:0] socket: 115 qp: -1
2025-11-11T17:21:36.811001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519231727094518:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:36.811047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:36.812263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094516:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.821018Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519231727094506:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:36.821281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519227432127129:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.821390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:36.821442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519227432127129:2048] [node 1] ICP22 created new session: [5:7571519231727094521:2048]
2025-11-11T17:21:36.821488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519231727094521:2048] [node 1] ICS09 handshake done sender: [5:7571519231727094506:2118] self: [5:7571519231727094510:0] peer: [1:7571519234203905550:0] socket: 90 qp: -1
2025-11-11T17:21:36.821528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519231727094521:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:36.821608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:36.822754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094520:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.834321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519231727094511:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:36.855419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519231727094489:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.855556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:36.855603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519231727094489:2108] [node 4] ICP22 created new session: [5:7571519231727094523:2108]
2025-11-11T17:21:36.855655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519231727094523:2108] [node 4] ICS09 handshake done sender: [5:7571519231727094511:2119] self: [5:7571519231727094517:0] peer: [4:7571519231428622756:0] socket: 133 qp: -1
2025-11-11T17:21:36.855696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519231727094523:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:36.855765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:36.869962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519231727094490:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(39865637ns) {HandleHandshakeStatus:329(32857994ns) {DropIncomingHandshake:441(17825ns) DropOutgoingHandshake:456(487ns) SwitchToState:178(603ns) ProcessPendingSessionEvents:495(484606ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 432901ns max# 54673ns}}}
2025-11-11T17:21:36.597956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519234203905566:2107] [node 4] ICS09 handshake done sender: [1:7571519234203905542:2121] self: [1:7571519234203905541:0] peer: [4:7571519231428622764:0] socket: 85 qp: -1
2025-11-11T17:21:36.767619Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519231428622770:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:36.733807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519233281917561:2127] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.845763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519234928511420:2108] [node 5] ICS09 handshake done sender: [2:7571519234928511400:2116] self: [2:7571519234928511398:0] peer: [5:7571519231727094509:0] socket: 106 qp: -1
2025-11-11T17:21:36.597989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519234203905566:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:36.598025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:36.651258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519234203905563:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43047
2025-11-11T17:21:36.678300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905563:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.698510Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905549:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.698539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519234203905549:2123] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.726169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519234203905537:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(83772277ns) {HandleHandshakeStatus:329(83768738ns) {DropIncomingHandshake:441(13956ns) DropOutgoingHandshake:456(337ns) SwitchToState:178(419ns) ProcessPendingSessionEvents:495(20391225ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20328095ns max# 19469149ns}}}
2025-11-11T17:21:36.742629Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519234203905549:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:36.765700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519229908938168:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.765759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:36.765819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519229908938168:2049] [node 3] ICP22 created new session: [1:7571519234203905571:2049]
2025-11-11T17:21:36.765870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519234203905571:2049] [node 3] ICS09 handshake done sender: [1:7571519234203905549:2123] self: [1:7571519234203905548:0] peer: [3:7571519233281917576:0] socket: 109 qp: -1
2025-11-11T17:21:36.765908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519234203905571:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:36.765950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:36.785833Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519234203905551:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.785876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519234203905551:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.797992Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519234203905551:2124] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:36.834502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519234203905535:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.851438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:36.862763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519234203905574:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43047
2025-11-11T17:21:36.863670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519234203905535:2108] [node 5] ICP22 created new session: [1:7571519234203905575:2108]
2025-11-11T17:21:36.887000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519234203905575:2108] [node 5] ICS09 handshake done sender: [1:7571519234203905551:2124] self: [1:7571519234203905550:0] peer: [5:7571519231727094510:0] socket: 79 qp: -1
2025-11-11T17:21:36.887992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519234203905575:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:36.897860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:36.742101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519233281917547:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.768746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519227133655388:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(53085015ns) {HandleHandshakeStatus:329(53081421ns) {DropIncomingHandshake:441(929ns) DropOutgoingHandshake:456(762ns) SwitchToState:178(1104ns) ProcessPendingSessionEvents:495(1879ns)}}
2025-11-11T17:21:36.845832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519234928511420:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:36.933959Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.794955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519227133655390:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.845896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:36.847500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519234928511386:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(26407258ns) {HandleHandshakeStatus:329(26399706ns) {DropIncomingHandshake:441(762ns) DropOutgoingHandshake:456(919ns) SwitchToState:178(1031ns) ProcessPendingSessionEvents:495(588432ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 524843ns max# 55693ns}}}
2025-11-11T17:21:36.941025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094525:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.932119Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:36.742205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:36.795120Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:36.954696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519233614292644:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.955017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519229319325255:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519233614292640:2120] held: no
2025-11-11T17:21:36.958042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519231727094525:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.958207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519227432127131:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.959991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519231727094527:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64249
2025-11-11T17:21:36.960753Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519231727094525:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:36.961103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519227432127131:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.742257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519233281917547:2107] [node 4] ICP22 created new session: [3:7571519233281917580:2107]
2025-11-11T17:21:36.951587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519234928511387:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(304669620ns) {HandleHandshakeStatus:329(304665968ns) {DropIncomingHandshake:441(26146ns) DropOutgoingHandshake:456(215ns) SwitchToState:178(850ns) ProcessPendingSessionEvents:495(276289074ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 256235090ns max# 18286805ns}}}
2025-11-11T17:21:36.961203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:36.966523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519233614292610:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.795987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:36.956777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519234928511397:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.742305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519233281917580:2107] [node 4] ICS09 handshake done sender: [3:7571519233281917559:2126] self: [3:7571519233281917558:0] peer: [4:7571519231428622776:0] socket: 116 qp: -1
2025-11-11T17:21:36.961250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519227432127131:2050] [node 3] ICP22 created new session: [5:7571519231727094528:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.798256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519227133655390:2050] [node 3] ICP22 created new session: [4:7571519231428622779:2050]
2025-11-11T17:21:36.961298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519231727094528:2050] [node 3] ICS09 handshake done sender: [5:7571519231727094525:2123] self: [5:7571519231727094526:0] peer: [3:7571519233281917574:0] socket: 99 qp: -1
2025-11-11T17:21:36.961333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519231727094528:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:36.961365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:36.956817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519234928511397:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.798306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519231428622779:2050] [node 3] ICS09 handshake done sender: [4:7571519231428622770:2120] self: [4:7571519231428622776:0] peer: [3:7571519233281917558:0] socket: 120 qp: -1
2025-11-11T17:21:36.742343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519233281917580:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:37.000976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519229908938168:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:37.001100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519233614292647:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:36.986910Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519234928511397:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:36.999068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519230633544027:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.999174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:36.999229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519230633544027:2049] [node 3] ICP22 created new session: [2:7571519234928511424:2049]
2025-11-11T17:21:36.999290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519234928511424:2049] [node 3] ICS09 handshake done sender: [2:7571519234928511397:2115] self: [2:7571519234928511396:0] peer: [3:7571519233281917591:0] socket: 134 qp: -1
2025-11-11T17:21:36.999332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519234928511424:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:36.999387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:36.742400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:36.798336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519231428622779:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:37.008654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519233614292609:2103] [node 5] ICP35 event processing took too much time PendingConnection:278(214126918ns) {HandleHandshakeStatus:329(214124192ns) {DropIncomingHandshake:441(883ns) DropOutgoingHandshake:456(6062ns) SwitchToState:178(708ns) ProcessPendingSessionEvents:495(184250192ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 184051823ns max# 16000862ns}}}
2025-11-11T17:21:36.786528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519233281917583:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12591
2025-11-11T17:21:36.801103Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519233281917561:2127] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:36.838441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519233281917556:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.838539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:36.838605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519233281917556:2118] [node 6] ICP22 created new session: [3:7571519233281917586:2118]
2025-11-11T17:21:36.838656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519233281917586:2118] [node 6] ICS09 handshake done sender: [3:7571519233281917561:2127] self: [3:7571519233281917560:0] peer: [6:7571519233614292643:0] socket: 126 qp: -1
2025-11-11T17:21:36.798395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:37.008776Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519233614292640:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:37.049677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519234203905535:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(199695901ns) {HandleHandshakeStatus:329(199691084ns) {DropIncomingHandshake:441(28436ns) DropOutgoingHandshake:456(546ns) SwitchToState:178(617ns) ProcessPendingSessionEvents:495(108357620ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 105810917ns max# 16537894ns}}}
2025-11-11T17:21:36.838699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519233281917586:2118] [node 6] ICS10 traffic start
2025-11-11T17:21:37.042828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519229319325255:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.799093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519227133655390:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:36.838763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:36.809831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519231428622781:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:371
2025-11-11T17:21:36.840032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519233281917568:2131] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:36.849856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519233281917585:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12591
2025-11-11T17:21:36.875219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519228986950180:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.875345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:36.875416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519228986950180:2048] [node 1] ICP22 created new session: [3:7571519233281917588:2048]
2025-11-11T17:21:36.875475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519233281917588:2048] [node 1] ICS09 handshake done sender: [3:7571519233281917568:2131] self: [3:7571519233281917576:0] peer: [1:7571519234203905548:0] socket: 117 qp: -1
2025-11-11T17:21:36.875515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519233281917588:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:36.875581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:36.925751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917585:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.949985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519228986950181:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:36.958631Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519233281917575:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.958658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519233281917575:2134] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:37.042919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:36.959702Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519233281917575:2134] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:36.998521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519233281917557:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:37.042984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519229319325255:2050] [node 3] ICP22 created new session: [6:7571519237909259945:2050]
2025-11-11T17:21:37.045328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519237909259946:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-11T17:21:37.027371Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:37.028633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519233281917593:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12591
2025-11-11T17:21:37.055259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519237909259945:2050] [node 3] ICS09 handshake done sender: [6:7571519233614292640:2120] self: [6:7571519233614292643:0] peer: [3:7571519233281917560:0] socket: 127 qp: -1
2025-11-11T17:21:37.058249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519237909259945:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:37.029110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519228986950181:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519233281917585:2136] held: no
2025-11-11T17:21:37.029913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:37.058302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:37.060067Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519233614292644:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:37.036116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519233281917557:2114] [node 5] ICP22 created new session: [3:7571519237576884890:2114]
2025-11-11T17:21:37.041444Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519233281917585:2136] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:37.062844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519229319325255:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18436685ns) {HandleHandshakeStatus:329(18433499ns) {DropIncomingHandshake:441(1029ns) DropOutgoingHandshake:456(818ns) SwitchToState:178(1796ns) ProcessPendingSessionEvents:495(1391ns)}}
2025-11-11T17:21:37.043536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519228986950181:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:37.056207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519237576884890:2114] [node 5] ICS09 handshake done sender: [3:7571519233281917575:2134] self: [3:7571519233281917574:0] peer: [5:7571519231727094526:0] socket: 93 qp: -1
2025-11-11T17:21:37.057967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519237576884890:2114] [node 5] ICS10 traffic start
2025-11-11T17:21:37.058032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:37.059753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:37.060336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519228986950181:2049] [node 2] ICP22 created new session: [3:7571519237576884892:2049]
2025-11-11T17:21:37.060416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519237576884892:2049] [node 2] ICS09 handshake done sender: [3:7571519233281917585:2136] self: [3:7571519233281917591:0] peer: [2:7571519234928511396:0] socket: 136 qp: -1
2025-11-11T17:21:37.060455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519237576884892:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:37.060497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:37.060733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519228986950181:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17125868ns) {HandleHandshakeStatus:329(17122051ns) {DropIncomingHandshake:441(517ns) DropOutgoingHandshake:456(700ns) SwitchToState:178(623ns) ProcessPendingSessionEvents:495(616ns)}}
2025-11-11T17:21:37.063603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519227432127131:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(79754428ns) {HandleHandshakeStatus:329(79752284ns) {DropIncomingHandshake:441(626ns) DropOutgoingHandshake:456(9802ns) SwitchToState:178(627ns) ProcessPendingSessionEvents:495(63310882ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 63205130ns max# 15122266ns}}}
2025-11-11T17:21:36.812524Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519231428622757:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:37.073209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519233281917557:2114] [node 5] ICP35 event processing took too much time PendingConnection:278(66083557ns) {HandleHandshakeStatus:329(66079492ns) {DropIncomingHandshake:441(14610ns) DropOutgoingHandshake:456(693ns) SwitchToState:178(743ns) ProcessPendingSessionEvents:495(4724056ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4666957ns max# 2299589ns}}}
2025-11-11T17:21:36.812551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519231428622757:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:37.067179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519233614292610:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:36.813661Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519231428622757:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:36.857940Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519231428622781:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.864608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519231428622748:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:36.864661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:36.879632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519231428622748:2108] [node 5] ICP22 created new session: [4:7571519231428622783:2108]
2025-11-11T17:21:36.890588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519231428622783:2108] [node 5] ICS09 handshake done sender: [4:7571519231428622757:2114] self: [4:7571519231428622756:0] peer: [5:7571519231727094517:0] socket: 95 qp: -1
2025-11-11T17:21:36.898767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519231428622783:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:36.901937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:36.991024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519231428622748:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(132106298ns) {HandleHandshakeStatus:329(132102544ns) {DropIncomingHandshake:441(1886ns) DropOutgoingHandshake:456(649ns) SwitchToState:178(1078ns) ProcessPendingSessionEvents:495(71821288ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 66869771ns max# 19706483ns}}}
2025-11-11T17:21:36.993818Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519231428622765:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:36.993866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519231428622765:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:36.998896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519227133655389:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(372020401ns) {HandleHandshakeStatus:329(372016220ns) {DropIncomingHandshake:441(6846ns) DropOutgoingHandshake:456(5631481ns) SwitchToState:178(2902ns) ProcessPendingSessionEvents:495(294122577ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 293831058ns max# 16044978ns}}}
2025-11-11T17:21:37.001435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519231428622748:2108] [node 5] ICP18 handshake (actor [4:7571519231428622781:2122]) is too old
2025-11-11T17:21:37.038048Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519231428622765:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:37.059216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519231428622747:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:37.059912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:37.059964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519231428622747:2109] [node 6] ICP22 created new session: [4:7571519235723590082:2109]
2025-11-11T17:21:37.061393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519235723590082:2109] [node 6] ICS09 handshake done sender: [4:7571519231428622765:2117] self: [4:7571519231428622763:0] peer: [6:7571519233614292648:0] socket: 128 qp: -1
2025-11-11T17:21:37.061434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519235723590082:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:37.061470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:37.082350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519231428622747:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(22394527ns) {HandleHandshakeStatus:329(22392225ns) {DropIncomingHandshake:441(15998ns) DropOutgoingHandshake:456(841ns) SwitchToState:178(666ns) ProcessPendingSessionEvents:495(19680822ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19606653ns max# 5731790ns}}}
2025-11-11T17:21:37.083235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:37.086862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519233614292610:2106] [node 4] ICP22 created new session: [6:7571519237909259948:2106]
2025-11-11T17:21:37.090413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519237909259948:2106] [node 4] ICS09 handshake done sender: [6:7571519233614292644:2122] self: [6:7571519233614292648:0] peer: [4:7571519231428622763:0] socket: 94 qp: -1
2025-11-11T17:21:37.091447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519237909259948:2106] [node 4] ICS10 traffic start
2025-11-11T17:21:37.088515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519233281917547:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(303629108ns) {HandleHandshakeStatus:329(303626635ns) {DropIncomingHandshake:441(13045ns) DropOutgoingHandshake:456(652ns) SwitchToState:178(1229ns) ProcessPendingSessionEvents:495(284158832ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 283865418ns max# 26787756ns}}}
2025-11-11T17:21:37.091801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:37.093407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519233614292610:2106] [node 4] ICP35 event processing took too much time PendingConnection:278(27322177ns) {HandleHandshakeStatus:329(25475735ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(9116ns) SwitchToState:178(680ns) ProcessPendingSessionEvents:495(709123ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 655149ns max# 223195ns}}}
2025-11-11T17:21:37.147419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:37.147679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519234928511416:2048] [node 1] ICS01 socket: 86 reason# EndOfStream
2025-11-11T17:21:37.147720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519230633544026:2048] [node 1] ICP30 unregister session Session# [2:7571519234928511416:2048] VirtualId# [2:7571519234928511405:0]
2025-11-11T17:21:37.147765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519234928511416:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.147856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:37.147901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:37.150136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:37.150371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519233281917588:2048] [node 1] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-11T17:21:37.150406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519233281917588:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.150488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:37.150528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519233281917588:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:37.153848Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:21:37.160005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:37.161582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:37.160656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:37.170300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519233614292633:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-11-11T17:21:37.170359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519229319325253:2048] [node 1] ICP30 unregister session Session# [6:7571519233614292633:2048] VirtualId# [6:7571519233614292630:0]
2025-11-11T17:21:37.170402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519233614292633:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.170540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:37.170592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:37.154421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519228986950180:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:37.154463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519233281917588:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:37.155121Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Couldn't connect to any resolved address
2025-11-11T17:21:37.155448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519228986950180:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:37.155504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519233281917588:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:37.155538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519228986950180:2048] [node 1] ICP30 unregister session Session# [3:7571519233281917588:2048] VirtualId# [3:7571519233281917576:0]
2025-11-11T17:21:37.155647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:37.155680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519228986950180:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:37.155693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:37.169920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519228986950180:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:37.171739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519231428622775:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-11-11T17:21:37.171678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519231727094521:2048] [node 1] ICS01 socket: 90 reason# EndOfStream
2025-11-11T17:21:37.171734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519227432127129:2048] [node 1] ICP30 unregister session Session# [5:7571519231727094521:2048] VirtualId# [5:7571519231727094510:0]
2025-11-11T17:21:37.171759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519231727094521:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.171845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:37.171893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:37.171794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519227133655388:2048] [node 1] ICP30 unregister session Session# [4:7571519231428622775:2048] VirtualId# [4:7571519231428622764:0]
2025-11-11T17:21:37.171822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519231428622775:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.171889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:37.171927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:37.239594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:37.239839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519237576884892:2049] [node 2] ICS01 socket: 136 reason# EndOfStream
2025-11-11T17:21:37.239886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519228986950181:2049] [node 2] ICP30 unregister session Session# [3:7571519237576884892:2049] VirtualId# [3:7571519233281917591:0]
2025-11-11T17:21:37.239926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519237576884892:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.240019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:37.240072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:37.244665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:37.244839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519231428622771:2049] [node 2] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-11T17:21:37.244866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519231428622771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.244932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:37.244967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519231428622771:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:37.246383Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62324) Couldn't connect to any resolved address
2025-11-11T17:21:37.246480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519227133655389:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62324) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:37.246777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:37.246910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519231727094518:2049] [node 2] ICS01 socket: 115 reason# EndOfStream
2025-11-11T17:21:37.246940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519227432127130:2049] [node 2] ICP30 unregister session Session# [5:7571519231727094518:2049] VirtualId# [5:7571519231727094509:0]
2025-11-11T17:21:37.246963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519231727094518:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.247023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:37.247061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:37.247579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519231428622771:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:37.247625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519227133655389:2049] [node 2] ICP30 unregister session Session# [4:7571519231428622771:2049] VirtualId# [4:7571519231428622766:0]
2025-11-11T17:21:37.247721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:37.247766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519227133655389:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:37.247780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:37.250615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:37.250977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519233614292635:2049] [node 2] ICS01 socket: 98 reason# EndOfStream
2025-11-11T17:21:37.251022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519229319325254:2049] [node 2] ICP30 unregister session Session# [6:7571519233614292635:2049] VirtualId# [6:7571519233614292629:0]
2025-11-11T17:21:37.251051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519233614292635:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.251115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:37.251143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:37.257543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519227133655389:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:37.281813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:37.282063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519231727094528:2050] [node 3] ICS01 socket: 99 reason# EndOfStream
2025-11-11T17:21:37.282106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519227432127131:2050] [node 3] ICP30 unregister session Session# [5:7571519231727094528:2050] VirtualId# [5:7571519231727094526:0]
2025-11-11T17:21:37.282142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519231727094528:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.282242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:37.282294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:37.282393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:37.282569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519237909259945:2050] [node 3] ICS01 socket: 127 reason# EndOfStream
2025-11-11T17:21:37.282605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519229319325255:2050] [node 3] ICP30 unregister session Session# [6:7571519237909259945:2050] VirtualId# [6:7571519233614292643:0]
2025-11-11T17:21:37.282634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519237909259945:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.282691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:37.282731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:37.283348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:37.283553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519231428622779:2050] [node 3] ICS01 socket: 120 reason# EndOfStream
2025-11-11T17:21:37.283588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519227133655390:2050] [node 3] ICP30 unregister session Session# [4:7571519231428622779:2050] VirtualId# [4:7571519231428622776:0]
2025-11-11T17:21:37.283614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519231428622779:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.283673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:37.283711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:37.364146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:37.364424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519231727094523:2108] [node 4] ICS01 socket: 133 reason# EndOfStream
2025-11-11T17:21:37.364464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519231727094489:2108] [node 4] ICP30 unregister session Session# [5:7571519231727094523:2108] VirtualId# [5:7571519231727094517:0]
2025-11-11T17:21:37.364501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519231727094523:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.364592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:37.364639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:37.365443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:37.365667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519237909259948:2106] [node 4] ICS01 socket: 94 reason# EndOfStream
2025-11-11T17:21:37.365705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519233614292610:2106] [node 4] ICP30 unregister session Session# [6:7571519237909259948:2106] VirtualId# [6:7571519233614292648:0]
2025-11-11T17:21:37.365732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519237909259948:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.365798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:37.365832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:37.438608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:37.438843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519233614292645:2103] [node 5] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-11T17:21:37.438883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519233614292645:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:37.438968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:37.439004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519233614292645:2103] [node 5] ICS15 start handshake
2025-11-11T17:21:37.439959Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63994) Couldn't connect to any resolved address
2025-11-11T17:21:37.440062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519233614292609:2103] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63994) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:37.440170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519233614292645:2103] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:37.440207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519233614292609:2103] [node 5] ICP30 unregister session Session# [6:7571519233614292645:2103] VirtualId# [6:7571519233614292641:0]
2025-11-11T17:21:37.440307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:21:37.440331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519233614292609:2103] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63994) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:37.440341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63994) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:37.527362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519236290923094:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:37.527424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519236290923095:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:37.555791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519237489000781:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:37.555806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519237489000782:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:37.581646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519236576813601:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:37.585241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519236576813602:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:37.606618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519236310683046:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:37.606663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519236310683047:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:37.606703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519236310683048:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:37.633514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519237377056058:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:37.633553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519237377056059:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:37.633799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519237377056057:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:37.658115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519237577973851:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:37.658163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519237577973852:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:37.658191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519237577973853:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:37.854371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519237489000842:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:37.857300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519236290923156:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:37.920900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519237489000846:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:37.920365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519236290923158:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:37.928434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519237577973905:2089] [node 4] ICP01 ready to work
2025-11-11T17:21:37.931772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519236576813664:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:37.932180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519237377056121:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:37.935031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519236310683110:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:37.930779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519236290923159:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:37.935616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519237489000847:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:37.934978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519237577973906:2098] [node 5] ICP01 ready to work
2025-11-11T17:21:37.945441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519236576813665:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:37.947976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519236576813667:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:37.948457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519237377056122:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:37.951753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519236576813601:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(87895209ns) {Configure:104(87889888ns) {ProcessConfigured:135(2375403ns) {StartInitialHandshake:157(2372852ns) {DropHandshakes:469(2123ns) {DropIncomingHandshake:441(509ns) DropOutgoingHandshake:456(380ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:476(2383ns) {SwitchToState:178(874ns)}}}}}
2025-11-11T17:21:37.948664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519236310683111:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:37.954918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519237489000781:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(102823672ns) {Configure:104(102817182ns) {ProcessConfigured:135(2943698ns) {StartInitialHandshake:157(2941832ns) {DropHandshakes:469(235303ns) {DropIncomingHandshake:441(731ns) DropOutgoingHandshake:456(768ns)} GenerateSessionVirtualId:988(853ns) PrepareNewSessionHandshake:476(2985ns) {SwitchToState:178(1049ns)}}}}}
2025-11-11T17:21:37.961756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519236290923094:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(115981307ns) {Configure:104(114731493ns) {ProcessConfigured:135(943851ns) {StartInitialHandshake:157(941333ns) {DropHandshakes:469(517239ns) {DropIncomingHandshake:441(723ns) DropOutgoingHandshake:456(367ns)} GenerateSessionVirtualId:988(1353ns) PrepareNewSessionHandshake:476(2037ns) {SwitchToState:178(518ns)}}}}}
2025-11-11T17:21:37.965351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519237489000782:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(105968250ns) {Configure:104(105960596ns) {ProcessConfigured:135(3706438ns) {StartInitialHandshake:157(3704188ns) {DropHandshakes:469(2954ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(403ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:476(7145ns) {SwitchToState:178(5661ns)}}}}}
2025-11-11T17:21:37.990401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519237577973853:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(131009100ns) {Configure:104(130998432ns) {ProcessConfigured:135(9270193ns) {StartInitialHandshake:157(9266021ns) {DropHandshakes:469(3690ns) {DropIncomingHandshake:441(1181ns) DropOutgoingHandshake:456(638ns)} GenerateSessionVirtualId:988(1628ns) PrepareNewSessionHandshake:476(7512ns) {SwitchToState:178(3960ns)}}}}}
2025-11-11T17:21:37.991955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519236290923159:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(39320150ns) {Configure:104(39316976ns) {ProcessConfigured:135(296008ns) {StartInitialHandshake:157(293839ns) {DropHandshakes:469(1809ns) {DropIncomingHandshake:441(693ns) DropOutgoingHandshake:456(347ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:476(1706ns) {SwitchToState:178(662ns)}}}}}
2025-11-11T17:21:37.991732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519237577973852:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(75319557ns) {Configure:104(75308011ns) {ProcessConfigured:135(13796737ns) {StartInitialHandshake:157(13792916ns) {DropHandshakes:469(4461ns) {DropIncomingHandshake:441(1559ns) DropOutgoingHandshake:456(491ns)} GenerateSessionVirtualId:988(1119ns) PrepareNewSessionHandshake:476(5087ns) {SwitchToState:178(2504ns)}}}}}
2025-11-11T17:21:37.996997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519237489000842:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(42419864ns) {Configure:104(42413718ns) {ProcessConfigured:135(1159466ns) {StartInitialHandshake:157(1156795ns) {DropHandshakes:469(5070ns) {DropIncomingHandshake:441(557ns) DropOutgoingHandshake:456(2845ns)} GenerateSessionVirtualId:988(1056ns) PrepareNewSessionHandshake:476(4013ns) {SwitchToState:178(1860ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:37.999083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519236576813667:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(42422266ns) {Configure:104(42415792ns) {ProcessConfigured:135(485807ns) {StartInitialHandshake:157(482400ns) {DropHandshakes:469(1501ns) {DropIncomingHandshake:441(366ns) DropOutgoingHandshake:456(322ns)} GenerateSessionVirtualId:988(537ns) PrepareNewSessionHandshake:476(1815ns) {SwitchToState:178(712ns)}}}}}
2025-11-11T17:21:38.006022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519237489000846:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(45427745ns) {Configure:104(45424589ns) {ProcessConfigured:135(613650ns) {StartInitialHandshake:157(612169ns) {DropHandshakes:469(1134ns) {DropIncomingHandshake:441(276ns) DropOutgoingHandshake:456(241ns)} GenerateSessionVirtualId:988(364ns) PrepareNewSessionHandshake:476(1973ns) {SwitchToState:178(869ns)}}}}}
2025-11-11T17:21:38.015559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519237377056058:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(47082779ns) {Configure:104(47077315ns) {ProcessConfigured:135(1580081ns) {StartInitialHandshake:157(1578235ns) {DropHandshakes:469(610ns) {DropIncomingHandshake:441(213ns) DropOutgoingHandshake:456(229ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:476(1908ns) {SwitchToState:178(544ns)}}}}}
2025-11-11T17:21:37.999293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519236576813602:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(41350560ns) {Configure:104(41345184ns) {ProcessConfigured:135(1623348ns) {StartInitialHandshake:157(1620712ns) {DropHandshakes:469(1323ns) {DropIncomingHandshake:441(505ns) DropOutgoingHandshake:456(239ns)} GenerateSessionVirtualId:988(719ns) PrepareNewSessionHandshake:476(2010ns) {SwitchToState:178(563ns)}}}}}
2025-11-11T17:21:38.020386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519236290923158:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(43085448ns) {Configure:104(43073769ns) {ProcessConfigured:135(821649ns) {StartInitialHandshake:157(820330ns) {DropHandshakes:469(1097ns) {DropIncomingHandshake:441(312ns) DropOutgoingHandshake:456(277ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:476(2735ns) {SwitchToState:178(891ns)}}}}}
2025-11-11T17:21:38.026322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519237577973851:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(47325157ns) {Configure:104(47317600ns) {ProcessConfigured:135(5340937ns) {StartInitialHandshake:157(5117869ns) {DropHandshakes:469(3855ns) {DropIncomingHandshake:441(708ns) DropOutgoingHandshake:456(509ns)} GenerateSessionVirtualId:988(7672ns) PrepareNewSessionHandshake:476(4873ns) {SwitchToState:178(2156ns)}}}}}
2025-11-11T17:21:38.029956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519237577973905:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(48943003ns) {Configure:104(48938510ns) {ProcessConfigured:135(291999ns) {StartInitialHandshake:157(289013ns) {DropHandshakes:469(5278ns) {DropIncomingHandshake:441(241ns) DropOutgoingHandshake:456(395ns)} GenerateSessionVirtualId:988(731ns) PrepareNewSessionHandshake:476(1733ns) {SwitchToState:178(893ns)}}}}}
2025-11-11T17:21:38.000131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519236576813677:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1268
2025-11-11T17:21:38.009994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519237489000847:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38515230ns) {Configure:104(38510002ns) {ProcessConfigured:135(385243ns) {StartInitialHandshake:157(381300ns) {DropHandshakes:469(675ns) {DropIncomingHandshake:441(154ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(354ns) PrepareNewSessionHandshake:476(1940ns) {SwitchToState:178(832ns)}}}}}
2025-11-11T17:21:38.026262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519236310683047:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(40261525ns) {Configure:104(40256741ns) {ProcessConfigured:135(1688132ns) {StartInitialHandshake:157(1685924ns) {DropHandshakes:469(1168ns) {DropIncomingHandshake:441(306ns) DropOutgoingHandshake:456(243ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:476(1894ns) {SwitchToState:178(994ns)}}}}}
2025-11-11T17:21:38.014051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519237377056121:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45528078ns) {Configure:104(45522840ns) {ProcessConfigured:135(267197ns) {StartInitialHandshake:157(265160ns) {DropHandshakes:469(1234ns) {DropIncomingHandshake:441(375ns) DropOutgoingHandshake:456(229ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(4501ns) {SwitchToState:178(1067ns)}}}}}
2025-11-11T17:21:38.032041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519236290923095:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(67901780ns) {Configure:104(67893877ns) {ProcessConfigured:135(5441323ns) {StartInitialHandshake:157(5438895ns) {DropHandshakes:469(2409ns) {DropIncomingHandshake:441(560ns) DropOutgoingHandshake:456(587ns)} GenerateSessionVirtualId:988(627ns) PrepareNewSessionHandshake:476(2075ns) {SwitchToState:178(561ns)}}}}}
2025-11-11T17:21:38.029861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519236310683046:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(49267075ns) {Configure:104(49256105ns) {ProcessConfigured:135(8986653ns) {StartInitialHandshake:157(8983238ns) {DropHandshakes:469(1006ns) {DropIncomingHandshake:441(275ns) DropOutgoingHandshake:456(240ns)} GenerateSessionVirtualId:988(366ns) PrepareNewSessionHandshake:476(5521ns) {SwitchToState:178(3317ns)}}}}}
2025-11-11T17:21:38.009883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519236576813665:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(44443216ns) {Configure:104(44437818ns) {ProcessConfigured:135(5321363ns) {StartInitialHandshake:157(5319524ns) {DropHandshakes:469(1381ns) {DropIncomingHandshake:441(314ns) DropOutgoingHandshake:456(211ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:476(6829ns) {SwitchToState:178(820ns)}}}}}
2025-11-11T17:21:38.039720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519236310683048:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(64619178ns) {Configure:104(64614473ns) {ProcessConfigured:135(16157888ns) {StartInitialHandshake:157(16154193ns) {DropHandshakes:469(2767ns) {DropIncomingHandshake:441(582ns) DropOutgoingHandshake:456(527ns)} GenerateSessionVirtualId:988(889ns) PrepareNewSessionHandshake:476(3414ns) {SwitchToState:178(1904ns)}}}}}
2025-11-11T17:21:38.020325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519237377056059:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(48654946ns) {Configure:104(48650204ns) {ProcessConfigured:135(650096ns) {StartInitialHandshake:157(647957ns) {DropHandshakes:469(624ns) {DropIncomingHandshake:441(165ns) DropOutgoingHandshake:456(122ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:476(2202ns) {SwitchToState:178(1257ns)}}}}}
2025-11-11T17:21:38.043973Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519237489000851:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.050933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941223:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.052837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941224:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.044024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519237489000851:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.010052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519236576813664:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(45111888ns) {Configure:104(45107551ns) {ProcessConfigured:135(1116903ns) {StartInitialHandshake:157(1114001ns) {DropHandshakes:469(871ns) {DropIncomingHandshake:441(268ns) DropOutgoingHandshake:456(217ns)} GenerateSessionVirtualId:988(381ns) PrepareNewSessionHandshake:476(2499ns) {SwitchToState:178(973ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.040259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519236310683110:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(69395223ns) {Configure:104(69390672ns) {ProcessConfigured:135(8743966ns) {StartInitialHandshake:157(8740748ns) {DropHandshakes:469(3823ns) {DropIncomingHandshake:441(787ns) DropOutgoingHandshake:456(853ns)} GenerateSessionVirtualId:988(948ns) PrepareNewSessionHandshake:476(3411ns) {SwitchToState:178(1471ns)}}}}}
2025-11-11T17:21:38.029267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519237377056057:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(55004498ns) {Configure:104(54999311ns) {ProcessConfigured:135(10955190ns) {StartInitialHandshake:157(10952786ns) {DropHandshakes:469(1125ns) {DropIncomingHandshake:441(344ns) DropOutgoingHandshake:456(231ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:476(4885ns) {SwitchToState:178(2631ns)}}}}}
2025-11-11T17:21:38.038564Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519236576813677:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.057823Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519241872941223:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.057741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519236290923166:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20776
2025-11-11T17:21:38.063931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519241872941224:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.067153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519237577973853:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.067185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519237577973852:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.071385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650416:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.054165Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519237489000851:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:38.065020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519241783968157:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11122
2025-11-11T17:21:38.038798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519236576813602:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.073393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519236576813673:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.073429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519236576813673:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.076489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519240871780980:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1268
2025-11-11T17:21:38.078413Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519236576813672:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.078430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519236576813672:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.079273Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519236576813672:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:38.072119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650417:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
2025-11-11T17:21:38.075300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519236290923163:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.082359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.078445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519237489000782:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.074739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519240605650417:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.075337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519236290923163:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.086224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519236576813665:2108] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.078508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.086280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:38.087041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519236576813665:2108] [node 4] ICP22 created new session: [3:7571519240871780983:2108]
2025-11-11T17:21:38.088692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023427:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.086547Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519241872941227:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.074921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519236310683046:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.075422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519236290923166:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.080097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519237489000857:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.077756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519236310683111:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(48866482ns) {Configure:104(48853979ns) {ProcessConfigured:135(76113ns) {StartInitialHandshake:157(72281ns) {DropHandshakes:469(11684ns) {DropIncomingHandshake:441(671ns) DropOutgoingHandshake:456(618ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:476(2056ns) {SwitchToState:178(747ns)}}}}}
2025-11-11T17:21:38.077907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519240605650416:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.078138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519236310683048:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.088406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650426:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
2025-11-11T17:21:38.101866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650425:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
2025-11-11T17:21:38.113176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519240605650416:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:38.123523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519236310683048:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.097754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519237377056122:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20382550ns) {Configure:104(20376146ns) {ProcessConfigured:135(11380305ns) {StartInitialHandshake:157(11377291ns) {DropHandshakes:469(3082ns) {DropIncomingHandshake:441(1260ns) DropOutgoingHandshake:456(463ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:476(4145ns) {SwitchToState:178(1959ns)}}}}}
2025-11-11T17:21:38.092544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519237577973851:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.086642Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519236290923163:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.092957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519240871780978:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1268
2025-11-11T17:21:38.080134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519237489000857:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.118047Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519241672023427:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.125479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:38.093714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941232:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.093361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519240871780982:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1268
2025-11-11T17:21:38.111161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519240585890472:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.082057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519237489000782:2049] [node 3] ICP22 created new session: [2:7571519241783968161:2049]
2025-11-11T17:21:38.127442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519236310683048:2050] [node 3] ICP22 created new session: [4:7571519240605650428:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.121907Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519236290923165:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.097853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941230:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.101886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519240871780983:2108] [node 4] ICS09 handshake done sender: [3:7571519236576813672:2111] self: [3:7571519236576813669:0] peer: [4:7571519240605650424:0] socket: 88 qp: -1
2025-11-11T17:21:38.086543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519241783968161:2049] [node 3] ICS09 handshake done sender: [2:7571519237489000851:2112] self: [2:7571519237489000850:0] peer: [3:7571519240871780977:0] socket: 85 qp: -1
2025-11-11T17:21:38.143090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023428:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.088032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519241783968160:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11122
2025-11-11T17:21:38.135565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519240605650428:2050] [node 3] ICS09 handshake done sender: [4:7571519240605650416:2115] self: [4:7571519240605650424:0] peer: [3:7571519236576813669:0] socket: 100 qp: -1
2025-11-11T17:21:38.121940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519236290923165:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.088830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519241783968161:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:38.154371Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519241672023428:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.136169Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519241872941223:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.108117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519240871780983:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:38.138122Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519240605650426:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.129852Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519236290923165:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:38.136570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519237577973853:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.092660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:38.094319Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519241783968157:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.108175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:38.154788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519237377056059:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.138752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650429:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
2025-11-11T17:21:38.105808Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519237489000857:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.135510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519236290923156:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.136632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:38.108413Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519236576813673:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:38.167753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023434:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.143546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519240605650428:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:38.135557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.181896Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519241672023434:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.114445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519237489000847:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.147605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:38.141097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519237577973853:2050] [node 3] ICP22 created new session: [6:7571519241872941233:2050]
2025-11-11T17:21:38.109359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519240871780979:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1268
2025-11-11T17:21:38.143286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519236290923156:2107] [node 4] ICP22 created new session: [1:7571519240585890474:2107]
2025-11-11T17:21:38.199812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519237377056058:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.115763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519241783968159:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11122
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.118468Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519240871780978:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.168385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650427:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
2025-11-11T17:21:38.147164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519241872941233:2050] [node 3] ICS09 handshake done sender: [6:7571519241872941223:2115] self: [6:7571519241872941228:0] peer: [3:7571519236576813671:0] socket: 97 qp: -1
2025-11-11T17:21:38.147231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519236290923159:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.179613Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519240605650429:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.118236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.214634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023439:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.189045Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519240605650417:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:38.125561Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519236576813677:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.219435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.154602Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519236290923161:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.156218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519241872941233:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:38.125571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519237489000847:2109] [node 6] ICP22 created new session: [2:7571519241783968167:2109]
2025-11-11T17:21:38.133090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519236576813667:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.228334Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519241672023427:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:38.228944Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519241672023436:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.249913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023440:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.254991Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:38.256860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519237377056059:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.256929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:38.256986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519237377056059:2050] [node 3] ICP22 created new session: [5:7571519241672023441:2050]
2025-11-11T17:21:38.257043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519241672023441:2050] [node 3] ICS09 handshake done sender: [5:7571519241672023427:2115] self: [5:7571519241672023435:0] peer: [3:7571519236576813670:0] socket: 96 qp: -1
2025-11-11T17:21:38.257082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519241672023441:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:38.257121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:38.268984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519237377056057:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.270236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023444:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.274417Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519241672023444:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.275611Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519241672023434:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:38.209107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519236310683047:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.127216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519241783968167:2109] [node 6] ICS09 handshake done sender: [2:7571519237489000857:2114] self: [2:7571519237489000856:0] peer: [6:7571519241872941229:0] socket: 91 qp: -1
2025-11-11T17:21:38.164523Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519241872941224:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:38.209178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519236310683046:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.133182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:38.154644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519236290923161:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.168384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519237577973852:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.280234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519237377056058:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.216238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.156284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:38.129321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519237489000781:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.281162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519237377056121:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.133233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519236576813667:2109] [node 6] ICP22 created new session: [3:7571519240871780987:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.216327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519236310683046:2048] [node 1] ICP22 created new session: [4:7571519240605650434:2048]
2025-11-11T17:21:38.168412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:38.129482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519241783968166:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11122
2025-11-11T17:21:38.282641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.133276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519240871780987:2109] [node 6] ICS09 handshake done sender: [3:7571519236576813673:2112] self: [3:7571519236576813671:0] peer: [6:7571519241872941228:0] socket: 89 qp: -1
2025-11-11T17:21:38.133308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519240871780987:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:38.133365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:38.134489Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519240871780982:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.156342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519236290923159:2109] [node 6] ICP22 created new session: [1:7571519240585890476:2109]
2025-11-11T17:21:38.216382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519240605650434:2048] [node 1] ICS09 handshake done sender: [4:7571519240605650417:2116] self: [4:7571519240605650421:0] peer: [1:7571519236290923162:0] socket: 104 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.172670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:38.135925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519241783968167:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:38.286339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519237377056058:2049] [node 2] ICP22 created new session: [5:7571519241672023446:2049]
2025-11-11T17:21:38.179615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519237577973852:2049] [node 2] ICP22 created new session: [6:7571519241872941237:2049]
2025-11-11T17:21:38.189042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519241872941237:2049] [node 2] ICS09 handshake done sender: [6:7571519241872941224:2116] self: [6:7571519241872941229:0] peer: [2:7571519237489000856:0] socket: 98 qp: -1
2025-11-11T17:21:38.189091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519241872941237:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:38.197580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:38.249924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941234:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.272122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519237577973852:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(103030189ns) {HandleHandshakeStatus:329(103026971ns) {DropIncomingHandshake:441(694ns) DropOutgoingHandshake:456(6246ns) SwitchToState:178(595ns) ProcessPendingSessionEvents:495(47875333ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47787581ns max# 26256955ns}}}
2025-11-11T17:21:38.272737Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519241872941227:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:38.275687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941235:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.280722Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519241872941235:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.284628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519237577973851:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.284692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:38.284745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519237577973851:2048] [node 1] ICP22 created new session: [6:7571519241872941241:2048]
2025-11-11T17:21:38.284801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519241872941241:2048] [node 1] ICS09 handshake done sender: [6:7571519241872941227:2117] self: [6:7571519241872941231:0] peer: [1:7571519236290923164:0] socket: 101 qp: -1
2025-11-11T17:21:38.284840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519241872941241:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:38.284909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:38.298944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519237577973905:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.318113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941244:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.134531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519236576813602:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.216436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519240605650434:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:38.156409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519240585890476:2109] [node 6] ICS09 handshake done sender: [1:7571519236290923165:2112] self: [1:7571519236290923164:0] peer: [6:7571519241872941231:0] socket: 95 qp: -1
2025-11-11T17:21:38.134611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:38.135972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:38.216506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:38.260289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650436:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
2025-11-11T17:21:38.280054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519240605650429:2121] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:38.289359Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519240605650423:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.289409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519240605650423:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.290090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519236310683047:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.290177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:38.290227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519236310683047:2049] [node 2] ICP22 created new session: [4:7571519240605650438:2049]
2025-11-11T17:21:38.290279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519240605650438:2049] [node 2] ICS09 handshake done sender: [4:7571519240605650429:2121] self: [4:7571519240605650433:0] peer: [2:7571519237489000854:0] socket: 110 qp: -1
2025-11-11T17:21:38.290313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519240605650438:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:38.290374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:38.299895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519240605650441:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12583
2025-11-11T17:21:38.302538Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519240605650423:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:38.312235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519236310683111:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.312346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:38.312388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519236310683111:2108] [node 5] ICP22 created new session: [4:7571519240605650442:2108]
2025-11-11T17:21:38.312430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519240605650442:2108] [node 5] ICS09 handshake done sender: [4:7571519240605650423:2117] self: [4:7571519240605650422:0] peer: [5:7571519241672023447:0] socket: 121 qp: -1
2025-11-11T17:21:38.312469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519240605650442:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:38.312499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:38.321704Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519240605650415:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.321766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519240605650415:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.328317Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519240605650441:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.156450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519240585890476:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:38.292914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519241672023446:2049] [node 2] ICS09 handshake done sender: [5:7571519241672023434:2118] self: [5:7571519241672023438:0] peer: [2:7571519237489000858:0] socket: 132 qp: -1
2025-11-11T17:21:38.134653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519236576813602:2049] [node 2] ICP22 created new session: [3:7571519240871780989:2049]
2025-11-11T17:21:38.350654Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519241872941244:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.151178Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519241783968159:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.351796Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519240605650415:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:38.365839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519237577973906:2098] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.297503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023449:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.352874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519236310683110:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.352964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:38.353001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519236310683110:2109] [node 6] ICP22 created new session: [4:7571519240605650446:2109]
2025-11-11T17:21:38.353034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519240605650446:2109] [node 6] ICS09 handshake done sender: [4:7571519240605650415:2114] self: [4:7571519240605650414:0] peer: [6:7571519241872941243:0] socket: 138 qp: -1
2025-11-11T17:21:38.353064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519240605650446:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:38.360721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:38.362652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519236310683048:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(238273775ns) {HandleHandshakeStatus:329(238269924ns) {DropIncomingHandshake:441(925ns) DropOutgoingHandshake:456(5725ns) SwitchToState:178(536ns) ProcessPendingSessionEvents:495(179915721ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 179765147ns max# 27165750ns}}}
2025-11-11T17:21:38.151322Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519241783968160:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.134700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519240871780989:2049] [node 2] ICS09 handshake done sender: [3:7571519236576813677:2115] self: [3:7571519240871780977:0] peer: [2:7571519237489000850:0] socket: 87 qp: -1
2025-11-11T17:21:38.158973Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519240585890472:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.134729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519240871780989:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:38.299415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519241672023446:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:38.151836Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519241783968166:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.134782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:38.161490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519240585890478:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20776
2025-11-11T17:21:38.389995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941245:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.304927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:38.172240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519241783968165:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11122
2025-11-11T17:21:38.396473Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:38.165022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519240585890474:2107] [node 4] ICS09 handshake done sender: [1:7571519236290923163:2111] self: [1:7571519236290923162:0] peer: [4:7571519240605650421:0] socket: 94 qp: -1
2025-11-11T17:21:38.305759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519241672023448:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-11T17:21:38.145579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519236576813667:2109] [node 6] ICP18 handshake (actor [3:7571519240871780978:2116]) is too old
2025-11-11T17:21:38.180192Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 3] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:38.167958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519240585890474:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:38.397038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519237577973851:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17768016ns) {HandleHandshakeStatus:329(17765789ns) {DropIncomingHandshake:441(2257ns) DropOutgoingHandshake:456(6061ns) SwitchToState:178(564ns) ProcessPendingSessionEvents:495(13477888ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13409448ns max# 7279756ns}}}
2025-11-11T17:21:38.412669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519237577973853:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(215429310ns) {HandleHandshakeStatus:329(215426643ns) {DropIncomingHandshake:441(669ns) DropOutgoingHandshake:456(4870ns) SwitchToState:178(339ns) ProcessPendingSessionEvents:495(166498039ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 159687273ns max# 30478790ns}}}
2025-11-11T17:21:38.414769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519237577973905:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519241872941235:2121] held: no
2025-11-11T17:21:38.185786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519241783968173:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11122
2025-11-11T17:21:38.209653Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:38.217775Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519241783968157:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:38.218342Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519237489000859:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.218374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519237489000859:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.218928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519237489000781:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519241783968157:2116] held: no
2025-11-11T17:21:38.232683Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519237489000855:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.232711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519237489000855:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.233719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519237489000781:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.233797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:38.146958Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:38.168035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:38.170946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:38.172407Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519236290923161:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:38.186038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519236290923094:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.186116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:38.186168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519236290923094:2048] [node 2] ICP22 created new session: [1:7571519240585890482:2048]
2025-11-11T17:21:38.186211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519240585890482:2048] [node 2] ICS09 handshake done sender: [1:7571519236290923161:2110] self: [1:7571519236290923160:0] peer: [2:7571519241783968168:0] socket: 92 qp: -1
2025-11-11T17:21:38.186253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519240585890482:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:38.186285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:38.205490Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519240585890468:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.205529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519240585890468:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.263066Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519240585890478:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.276226Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519236290923168:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.276245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519236290923168:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.277259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519240585890485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20776
2025-11-11T17:21:38.277322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519240585890486:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20776
2025-11-11T17:21:38.280191Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519240585890468:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:38.280997Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519236290923168:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:38.293347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519236290923095:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.293409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:38.293457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519236290923095:2049] [node 3] ICP22 created new session: [1:7571519240585890489:2049]
2025-11-11T17:21:38.293501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519240585890489:2049] [node 3] ICS09 handshake done sender: [1:7571519240585890468:2115] self: [1:7571519240585890467:0] peer: [3:7571519240871780995:0] socket: 140 qp: -1
2025-11-11T17:21:38.293534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519240585890489:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:38.293573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:38.303236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519236290923158:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.306084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:38.306141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519236290923158:2108] [node 5] ICP22 created new session: [1:7571519240585890491:2108]
2025-11-11T17:21:38.306193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519240585890491:2108] [node 5] ICS09 handshake done sender: [1:7571519236290923168:2114] self: [1:7571519236290923167:0] peer: [5:7571519241672023443:0] socket: 137 qp: -1
2025-11-11T17:21:38.306249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519240585890491:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:38.306297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:38.402802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519236290923158:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(98767081ns) {HandleHandshakeStatus:329(98762195ns) {DropIncomingHandshake:441(263ns) DropOutgoingHandshake:456(209ns) SwitchToState:178(444ns) ProcessPendingSessionEvents:495(77188348ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75817212ns max# 12836479ns}}}
2025-11-11T17:21:38.414082Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:38.414398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519236290923159:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(154340664ns) {HandleHandshakeStatus:329(154337553ns) {DropIncomingHandshake:441(1333ns) DropOutgoingHandshake:456(730ns) SwitchToState:178(366ns) ProcessPendingSessionEvents:495(130514718ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 130355644ns max# 19664689ns}}}
2025-11-11T17:21:38.441133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519236290923159:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:38.351148Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:38.167254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519240871780993:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1268
2025-11-11T17:21:38.177675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519236576813674:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.177727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519236576813674:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.184250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519236576813665:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(97111481ns) {HandleHandshakeStatus:329(97108425ns) {DropIncomingHandshake:441(459ns) DropOutgoingHandshake:456(710ns) SwitchToState:178(324ns) ProcessPendingSessionEvents:495(57785512ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 57672054ns max# 9560926ns}}}
2025-11-11T17:21:38.184813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519236576813665:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:38.193049Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519240871780993:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.193184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519236576813601:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:38.197859Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519236576813674:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:38.219304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519236576813664:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.249809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:38.257234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519236576813664:2107] [node 5] ICP22 created new session: [3:7571519240871780996:2107]
2025-11-11T17:21:38.268218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519240871780996:2107] [node 5] ICS09 handshake done sender: [3:7571519236576813674:2113] self: [3:7571519236576813670:0] peer: [5:7571519241672023435:0] socket: 90 qp: -1
2025-11-11T17:21:38.269118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519240871780996:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:38.273604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:38.275419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519240871780997:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1268
2025-11-11T17:21:38.294161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519240871780993:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:38.300273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519236576813601:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.314758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:38.325195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519236576813601:2048] [node 1] ICP22 created new session: [3:7571519240871780999:2048]
2025-11-11T17:21:38.329309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519240871780999:2048] [node 1] ICS09 handshake done sender: [3:7571519240871780993:2120] self: [3:7571519240871780995:0] peer: [1:7571519240585890467:0] socket: 141 qp: -1
2025-11-11T17:21:38.333494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519240871780999:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:38.337405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:38.418193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519236576813664:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(180916313ns) {HandleHandshakeStatus:329(180911234ns) {DropIncomingHandshake:441(38679ns) DropOutgoingHandshake:456(271ns) SwitchToState:178(677ns) ProcessPendingSessionEvents:495(117074266ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 109091199ns max# 19649775ns}}}
2025-11-11T17:21:38.436689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519236576813601:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(135549714ns) {HandleHandshakeStatus:329(135546601ns) {DropIncomingHandshake:441(3150ns) DropOutgoingHandshake:456(12007ns) SwitchToState:178(869ns) ProcessPendingSessionEvents:495(67729190ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67635461ns max# 21717753ns}}}
2025-11-11T17:21:38.233865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519237489000781:2048] [node 1] ICP22 created new session: [2:7571519241783968176:2048]
2025-11-11T17:21:38.233911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519241783968176:2048] [node 1] ICS09 handshake done sender: [2:7571519241783968157:2116] self: [2:7571519241783968168:0] peer: [1:7571519236290923160:0] socket: 93 qp: -1
2025-11-11T17:21:38.461249Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519240585890486:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.365745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519241672023444:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:38.233945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519241783968176:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:38.234008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:38.262085Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519237489000859:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:38.274018Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519237489000855:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:38.280263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519237489000846:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.282305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:38.282359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519237489000846:2108] [node 5] ICP22 created new session: [2:7571519241783968178:2108]
2025-11-11T17:21:38.297737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519237489000842:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.297863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:38.297923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519237489000842:2107] [node 4] ICP22 created new session: [2:7571519241783968179:2107]
2025-11-11T17:21:38.297973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519241783968179:2107] [node 4] ICS09 handshake done sender: [2:7571519237489000855:2113] self: [2:7571519237489000854:0] peer: [4:7571519240605650433:0] socket: 135 qp: -1
2025-11-11T17:21:38.298010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519241783968179:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:38.298045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:38.299399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519241783968178:2108] [node 5] ICS09 handshake done sender: [2:7571519237489000859:2115] self: [2:7571519237489000858:0] peer: [5:7571519241672023438:0] socket: 126 qp: -1
2025-11-11T17:21:38.300275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519241783968178:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:38.300324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:38.403938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519237489000847:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(167460501ns) {HandleHandshakeStatus:329(167456315ns) {DropIncomingHandshake:441(1120ns) DropOutgoingHandshake:456(658ns) SwitchToState:178(612ns) ProcessPendingSessionEvents:495(130746068ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 120844651ns max# 12935962ns}}}
2025-11-11T17:21:38.409816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519237489000782:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(200209651ns) {HandleHandshakeStatus:329(200207073ns) {DropIncomingHandshake:441(293ns) DropOutgoingHandshake:456(519ns) SwitchToState:178(363ns) ProcessPendingSessionEvents:495(165602299ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 154965213ns max# 11890458ns}}}
2025-11-11T17:21:38.486868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519236290923095:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(192699774ns) {HandleHandshakeStatus:329(192697246ns) {DropIncomingHandshake:441(14014ns) DropOutgoingHandshake:456(600ns) SwitchToState:178(460ns) ProcessPendingSessionEvents:495(192290984ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 178256004ns max# 18780005ns}}}
2025-11-11T17:21:38.487885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519236290923158:2108] [node 5] ICP18 handshake (actor [1:7571519240585890486:2119]) is too old
2025-11-11T17:21:38.396467Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519241672023436:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:38.419066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519237377056121:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519241672023444:2122] held: no
2025-11-11T17:21:38.419674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519237377056121:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.419772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:38.419835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519237377056121:2108] [node 4] ICP22 created new session: [5:7571519241672023452:2108]
2025-11-11T17:21:38.419882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519241672023452:2108] [node 4] ICS09 handshake done sender: [5:7571519241672023444:2122] self: [5:7571519241672023447:0] peer: [4:7571519240605650422:0] socket: 147 qp: -1
2025-11-11T17:21:38.424412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519241672023452:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:38.424470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:38.425418Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519241672023432:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:38.425443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519241672023432:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:38.430089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519237377056057:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.430180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:38.430231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519237377056057:2048] [node 1] ICP22 created new session: [5:7571519241672023454:2048]
2025-11-11T17:21:38.430295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519241672023454:2048] [node 1] ICS09 handshake done sender: [5:7571519241672023436:2119] self: [5:7571519241672023443:0] peer: [1:7571519236290923167:0] socket: 99 qp: -1
2025-11-11T17:21:38.430333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519241672023454:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:38.430406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:38.484794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519237377056058:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(161618993ns) {HandleHandshakeStatus:329(161616555ns) {DropIncomingHandshake:441(1110ns) DropOutgoingHandshake:456(5440ns) SwitchToState:178(612ns) ProcessPendingSessionEvents:495(119088757ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 107944237ns max# 13044774ns}}}
2025-11-11T17:21:38.490699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519237377056059:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(199699329ns) {HandleHandshakeStatus:329(199695897ns) {DropIncomingHandshake:441(1241ns) DropOutgoingHandshake:456(14724ns) SwitchToState:178(982ns) ProcessPendingSessionEvents:495(185693888ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 181521509ns max# 18122316ns}}}
2025-11-11T17:21:38.490868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519237377056059:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:38.490933Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519241672023432:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:38.492708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519237489000846:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(208194923ns) {HandleHandshakeStatus:329(208191710ns) {DropIncomingHandshake:441(14900ns) DropOutgoingHandshake:456(459ns) SwitchToState:178(379ns) ProcessPendingSessionEvents:495(177589178ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 175682790ns max# 18257014ns}}}
2025-11-11T17:21:38.505934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519236290923156:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(251811219ns) {HandleHandshakeStatus:329(251808059ns) {DropIncomingHandshake:441(1026ns) DropOutgoingHandshake:456(313ns) SwitchToState:178(530ns) ProcessPendingSessionEvents:495(201579476ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 199417979ns max# 18967269ns}}}
2025-11-11T17:21:38.510182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519237377056122:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:38.506890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519236290923156:2107] [node 4] ICP18 handshake (actor [1:7571519240585890472:2116]) is too old
2025-11-11T17:21:38.513930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:38.516004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519237577973905:2089] [node 4] ICP35 event processing took too much time PendingConnection:278(79766948ns) {HandleHandshakeStatus:395(79763141ns) {LogHandshakeFail:472(15807ns) {UpdateErrorStateLog:544(5640ns)} DropOutgoingHandshake:456(1580ns) ProcessPendingSessionEvents:495(75071462ns) {unordered DropSessionEvent:512 num# 100 sum# 24300242ns max# 9695433ns}}}
2025-11-11T17:21:38.530810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519237489000847:2109] [node 6] ICP18 handshake (actor [2:7571519241783968159:2117]) is too old
2025-11-11T17:21:38.526582Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24683) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:38.515100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519237377056122:2109] [node 6] ICP22 created new session: [5:7571519241672023457:2109]
2025-11-11T17:21:38.532451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519237577973906:2098] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24683) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519241872941244:2122] held: no
2025-11-11T17:21:38.537709Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519241872941235:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:38.539352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519241872941247:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-11T17:21:38.543533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519237577973905:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.543626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:38.543677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519237577973905:2089] [node 4] ICP22 created new session: [6:7571519241872941248:2089]
2025-11-11T17:21:38.543722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519241872941248:2089] [node 4] ICS09 handshake done sender: [6:7571519241872941235:2121] self: [6:7571519241872941243:0] peer: [4:7571519240605650414:0] socket: 128 qp: -1
2025-11-11T17:21:38.543765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519241872941248:2089] [node 4] ICS10 traffic start
2025-11-11T17:21:38.543796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:38.518059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519237377056057:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22437061ns) {HandleHandshakeStatus:329(22433226ns) {DropIncomingHandshake:441(1066ns) DropOutgoingHandshake:456(4679ns) SwitchToState:178(850ns) ProcessPendingSessionEvents:495(21928512ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21847311ns max# 10226498ns}}}
2025-11-11T17:21:38.526731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519241672023457:2109] [node 6] ICS09 handshake done sender: [5:7571519241672023432:2117] self: [5:7571519241672023429:0] peer: [6:7571519241872941246:0] socket: 154 qp: -1
2025-11-11T17:21:38.530320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519241672023457:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:38.532734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:38.576668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519237489000782:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:38.579484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519237377056122:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(68644925ns) {HandleHandshakeStatus:329(68641462ns) {DropIncomingHandshake:441(25172ns) DropOutgoingHandshake:456(790ns) SwitchToState:178(537ns) ProcessPendingSessionEvents:495(11791084ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11710347ns max# 9393966ns}}}
2025-11-11T17:21:38.579212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519236310683110:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(225432003ns) {HandleHandshakeStatus:329(225429704ns) {DropIncomingHandshake:441(11880ns) DropOutgoingHandshake:456(224ns) SwitchToState:178(291ns) ProcessPendingSessionEvents:495(183854994ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 183643388ns max# 15953796ns}}}
2025-11-11T17:21:38.581523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519237577973906:2098] [node 5] ICP35 event processing took too much time PendingConnection:278(50757809ns) {HandleHandshakeStatus:395(50753988ns) {LogHandshakeFail:472(17200ns) {UpdateErrorStateLog:544(5641ns)} DropOutgoingHandshake:456(1561ns) ProcessPendingSessionEvents:495(48317638ns) {unordered DropSessionEvent:512 num# 100 sum# 48264440ns max# 14067306ns}}}
2025-11-11T17:21:38.581694Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519241872941244:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:38.582155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519237577973906:2098] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:38.582227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:38.582272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519237577973906:2098] [node 5] ICP22 created new session: [6:7571519241872941250:2098]
2025-11-11T17:21:38.582318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519241872941250:2098] [node 5] ICS09 handshake done sender: [6:7571519241872941244:2122] self: [6:7571519241872941246:0] peer: [5:7571519241672023429:0] socket: 133 qp: -1
2025-11-11T17:21:38.582355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519241872941250:2098] [node 5] ICS10 traffic start
2025-11-11T17:21:38.582392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:38.633008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:38.633295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519241783968176:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-11T17:21:38.633338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519237489000781:2048] [node 1] ICP30 unregister session Session# [2:7571519241783968176:2048] VirtualId# [2:7571519241783968168:0]
2025-11-11T17:21:38.633374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519241783968176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.633455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:38.633494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:38.635937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:38.636149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519240871780999:2048] [node 1] ICS01 socket: 141 reason# EndOfStream
2025-11-11T17:21:38.636191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519236576813601:2048] [node 1] ICP30 unregister session Session# [3:7571519240871780999:2048] VirtualId# [3:7571519240871780995:0]
2025-11-11T17:21:38.636228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519240871780999:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.636302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:38.636341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:38.639571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:38.639818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519240605650434:2048] [node 1] ICS01 socket: 104 reason# EndOfStream
2025-11-11T17:21:38.639857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519236310683046:2048] [node 1] ICP30 unregister session Session# [4:7571519240605650434:2048] VirtualId# [4:7571519240605650421:0]
2025-11-11T17:21:38.639901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519240605650434:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.639978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:38.640021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:38.643180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:38.643348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519241672023454:2048] [node 1] ICS07 socket disconnect 99 reason# EndOfStream
2025-11-11T17:21:38.643398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519241672023454:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.643467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:38.643511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519241672023454:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:38.643194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:38.643418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519241872941241:2048] [node 1] ICS07 socket disconnect 101 reason# EndOfStream
2025-11-11T17:21:38.643478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519241872941241:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.643559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:38.643594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519241872941241:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:38.645065Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address
2025-11-11T17:21:38.647296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519237377056057:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:38.647271Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address
2025-11-11T17:21:38.647404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519241672023454:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:38.647997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519237377056057:2048] [node 1] ICP30 unregister session Session# [5:7571519241672023454:2048] VirtualId# [5:7571519241672023443:0]
2025-11-11T17:21:38.648099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:38.648148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519237377056057:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.648170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.648001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519237577973851:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:38.648046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519241872941241:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:38.648078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519237577973851:2048] [node 1] ICP30 unregister session Session# [6:7571519241872941241:2048] VirtualId# [6:7571519241872941231:0]
2025-11-11T17:21:38.648193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:38.648228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519237577973851:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.648242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.657323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519237577973851:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:38.658366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519237377056057:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:38.734505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:38.734722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519240871780989:2049] [node 2] ICS01 socket: 87 reason# EndOfStream
2025-11-11T17:21:38.734769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519236576813602:2049] [node 2] ICP30 unregister session Session# [3:7571519240871780989:2049] VirtualId# [3:7571519240871780977:0]
2025-11-11T17:21:38.734807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519240871780989:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.734890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:38.734939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:38.741810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:38.742050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519241872941237:2049] [node 2] ICS01 socket: 98 reason# EndOfStream
2025-11-11T17:21:38.742092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519237577973852:2049] [node 2] ICP30 unregister session Session# [6:7571519241872941237:2049] VirtualId# [6:7571519241872941229:0]
2025-11-11T17:21:38.742131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519241872941237:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.742214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:38.742256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:38.741862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:38.742060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519241672023446:2049] [node 2] ICS01 socket: 132 reason# EndOfStream
2025-11-11T17:21:38.742098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519237377056058:2049] [node 2] ICP30 unregister session Session# [5:7571519241672023446:2049] VirtualId# [5:7571519241672023438:0]
2025-11-11T17:21:38.742132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519241672023446:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.742220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:38.742264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:38.741811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:38.742085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519240605650438:2049] [node 2] ICS01 socket: 110 reason# EndOfStream
2025-11-11T17:21:38.742126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519236310683047:2049] [node 2] ICP30 unregister session Session# [4:7571519240605650438:2049] VirtualId# [4:7571519240605650433:0]
2025-11-11T17:21:38.742151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519240605650438:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.742231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:38.742271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:38.820407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:38.820663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519241872941233:2050] [node 3] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-11T17:21:38.820694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519241872941233:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.820802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:38.820851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519241872941233:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:38.821897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:38.821647Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address
2025-11-11T17:21:38.821906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519237577973853:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:38.821953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519241872941233:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:38.821985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519237577973853:2050] [node 3] ICP30 unregister session Session# [6:7571519241872941233:2050] VirtualId# [6:7571519241872941228:0]
2025-11-11T17:21:38.822096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:38.822126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519237577973853:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.822167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.822076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519240605650428:2050] [node 3] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-11T17:21:38.822107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519240605650428:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.822187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:38.822225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519240605650428:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:38.823271Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address
2025-11-11T17:21:38.823350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519236310683048:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:38.823412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519240605650428:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:38.823448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519236310683048:2050] [node 3] ICP30 unregister session Session# [4:7571519240605650428:2050] VirtualId# [4:7571519240605650424:0]
2025-11-11T17:21:38.830120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:38.821440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:38.830392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519236310683048:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.830419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.821650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519241672023441:2050] [node 3] ICS07 socket disconnect 96 reason# EndOfStream
2025-11-11T17:21:38.821674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519241672023441:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.821753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:38.821800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519241672023441:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:38.831720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519237577973853:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:38.831190Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address
2025-11-11T17:21:38.831445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519237377056059:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:38.831494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519241672023441:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:38.831527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519237377056059:2050] [node 3] ICP30 unregister session Session# [5:7571519241672023441:2050] VirtualId# [5:7571519241672023435:0]
2025-11-11T17:21:38.831604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:38.831628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519237377056059:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.831642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:38.839380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519236310683048:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:38.840710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519237377056059:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:38.894896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:38.895114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519241872941248:2089] [node 4] ICS01 socket: 128 reason# EndOfStream
2025-11-11T17:21:38.895174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519237577973905:2089] [node 4] ICP30 unregister session Session# [6:7571519241872941248:2089] VirtualId# [6:7571519241872941243:0]
2025-11-11T17:21:38.895211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519241872941248:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.895289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:38.895366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:38.897718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:38.897922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519241672023452:2108] [node 4] ICS01 socket: 147 reason# EndOfStream
2025-11-11T17:21:38.897968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519237377056121:2108] [node 4] ICP30 unregister session Session# [5:7571519241672023452:2108] VirtualId# [5:7571519241672023447:0]
2025-11-11T17:21:38.898000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519241672023452:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.898085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:38.898131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:38.985776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:38.986011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519241872941250:2098] [node 5] ICS01 socket: 133 reason# EndOfStream
2025-11-11T17:21:38.986054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519237577973906:2098] [node 5] ICP30 unregister session Session# [6:7571519241872941250:2098] VirtualId# [6:7571519241872941246:0]
2025-11-11T17:21:38.986090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519241872941250:2098] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:38.986189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:38.986233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:39.109732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519247616278890:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:39.109802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519247616278889:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:39.134174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519247041469420:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:39.134224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519247041469421:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:39.167443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519246636446254:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:39.167493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519246636446255:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:39.184127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519245900460992:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:39.184155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519245900460993:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:39.184172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519245900460994:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:39.210669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519244161318615:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:39.210714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519244161318617:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:39.210732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519244161318616:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:39.239310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519246806691506:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:39.239310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519246806691505:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:39.239352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519246806691507:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:39.342964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519246806691567:2106] [node 5] ICP01 ready to work
2025-11-11T17:21:39.354375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519247616278953:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.355800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519247616278951:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:39.430851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519247041469500:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:39.431852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519247041469501:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:39.398869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519247616278954:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:39.432602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318676:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:39.435811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519245900461075:2128] [node 6] ICP01 ready to work
2025-11-11T17:21:39.436043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519246636446335:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:39.458397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519246806691579:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:39.440331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519247041469502:2111] [node 6] ICP01 ready to work
2025-11-11T17:21:39.433854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519244161318679:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:39.460006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519245900461080:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:39.469696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519246636446336:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:39.434202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519244161318680:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:39.497241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519246636446337:2114] [node 6] ICP01 ready to work
2025-11-11T17:21:39.501521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519246806691507:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(69640371ns) {Configure:104(67872839ns) {ProcessConfigured:135(82975ns) {StartInitialHandshake:157(80009ns) {DropHandshakes:469(3169ns) {DropIncomingHandshake:441(1087ns) DropOutgoingHandshake:456(488ns)} GenerateSessionVirtualId:988(1045ns) PrepareNewSessionHandshake:476(2769ns) {SwitchToState:178(1127ns)}}}}}
2025-11-11T17:21:39.469700Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519244161318676:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.512465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519247041469503:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35146
2025-11-11T17:21:39.515486Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519247616278953:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(33177951ns) {Configure:104(33171848ns) {ProcessConfigured:135(2004156ns) {StartInitialHandshake:157(2001521ns) {DropHandshakes:469(3338ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(649ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:476(3282ns) {SwitchToState:178(960ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.535688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519245900461080:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(37990233ns) {Configure:104(37986301ns) {ProcessConfigured:135(8291541ns) {StartInitialHandshake:157(8288291ns) {DropHandshakes:469(3229ns) {DropIncomingHandshake:441(680ns) DropOutgoingHandshake:456(660ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:476(4651ns) {SwitchToState:178(1902ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.542063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519244161318680:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(60137349ns) {Configure:104(60132489ns) {ProcessConfigured:135(13669344ns) {StartInitialHandshake:157(13667141ns) {DropHandshakes:469(1461ns) {DropIncomingHandshake:441(324ns) DropOutgoingHandshake:456(336ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:476(4393ns) {SwitchToState:178(1642ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.542525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318687:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.525898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519247041469421:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(80475765ns) {Configure:104(80468402ns) {ProcessConfigured:135(4516805ns) {StartInitialHandshake:157(4514405ns) {DropHandshakes:469(1639ns) {DropIncomingHandshake:441(348ns) DropOutgoingHandshake:456(410ns)} GenerateSessionVirtualId:988(555ns) PrepareNewSessionHandshake:476(1081865ns) {SwitchToState:178(3510ns)}}}}}
2025-11-11T17:21:39.525896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519246806691505:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(124643547ns) {Configure:104(124638012ns) {ProcessConfigured:135(4063049ns) {StartInitialHandshake:157(4060423ns) {DropHandshakes:469(1579ns) {DropIncomingHandshake:441(282ns) DropOutgoingHandshake:456(526ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:476(1710ns) {SwitchToState:178(834ns)}}}}}
2025-11-11T17:21:39.526212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519247616278890:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(159449418ns) {Configure:104(159443192ns) {ProcessConfigured:135(1598854ns) {StartInitialHandshake:157(1596451ns) {DropHandshakes:469(1431ns) {DropIncomingHandshake:441(389ns) DropOutgoingHandshake:456(284ns)} GenerateSessionVirtualId:988(739ns) PrepareNewSessionHandshake:476(1973ns) {SwitchToState:178(561ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.545747Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519244161318686:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.529020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519247041469420:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23961174ns) {Configure:104(23953877ns) {ProcessConfigured:135(47373ns) {StartInitialHandshake:157(45128ns) {DropHandshakes:469(880ns) {DropIncomingHandshake:441(384ns) DropOutgoingHandshake:456(91ns)} GenerateSessionVirtualId:988(264ns) PrepareNewSessionHandshake:476(1745ns) {SwitchToState:178(639ns)}}}}}
2025-11-11T17:21:39.528351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519246806691506:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(83495184ns) {Configure:104(82445943ns) {ProcessConfigured:135(2375899ns) {StartInitialHandshake:157(2369590ns) {DropHandshakes:469(1284ns) {DropIncomingHandshake:441(344ns) DropOutgoingHandshake:456(274ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:476(1552ns) {SwitchToState:178(491ns)}}}}}
2025-11-11T17:21:39.527109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519247616278889:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(92383825ns) {Configure:104(88139601ns) {ProcessConfigured:135(4045260ns) {StartInitialHandshake:157(4042426ns) {DropHandshakes:469(1428ns) {DropIncomingHandshake:441(366ns) DropOutgoingHandshake:456(255ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:476(2470ns) {SwitchToState:178(748ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.545776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519244161318686:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.554279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519245900461081:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.597906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519246636446337:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(76159089ns) {Configure:104(76156373ns) {ProcessConfigured:135(19875113ns) {StartInitialHandshake:157(16073165ns) {DropHandshakes:469(4468ns) {DropIncomingHandshake:441(635ns) DropOutgoingHandshake:456(841ns)} GenerateSessionVirtualId:988(2041ns) PrepareNewSessionHandshake:476(4901ns) {SwitchToState:178(2313ns)}}}}}
2025-11-11T17:21:39.546671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519247041469500:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(48225476ns) {Configure:104(48219411ns) {ProcessConfigured:135(17294093ns) {StartInitialHandshake:157(17290997ns) {DropHandshakes:469(894ns) {DropIncomingHandshake:441(185ns) DropOutgoingHandshake:456(111ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:476(6884ns) {SwitchToState:178(1842ns)}}}}}
2025-11-11T17:21:39.558881Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519244161318687:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.543134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691590:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.559627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519245900461085:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.611570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519246636446255:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(89031859ns) {Configure:104(89024593ns) {ProcessConfigured:135(79353ns) {StartInitialHandshake:157(76247ns) {DropHandshakes:469(3459ns) {DropIncomingHandshake:441(999ns) DropOutgoingHandshake:456(534ns)} GenerateSessionVirtualId:988(1492ns) PrepareNewSessionHandshake:476(2041ns) {SwitchToState:178(765ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.550410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519247616278955:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45297
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.545352Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519246806691590:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.600303Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519245900461083:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.613363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519246636446338:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46928
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.550326Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469503:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.568368Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519244161318686:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.589152Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278955:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.617805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519246636446254:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(76787365ns) {Configure:104(76783412ns) {ProcessConfigured:135(12526735ns) {StartInitialHandshake:157(12524836ns) {DropHandshakes:469(3957ns) {DropIncomingHandshake:441(509ns) DropOutgoingHandshake:456(872ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:476(3033ns) {SwitchToState:178(834ns)}}}}}
2025-11-11T17:21:39.600330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519245900461083:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.545488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519246806691567:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.574448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519244161318679:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(48786142ns) {Configure:104(48780327ns) {ProcessConfigured:135(23365798ns) {StartInitialHandshake:157(23364303ns) {DropHandshakes:469(2937ns) {DropIncomingHandshake:441(994ns) DropOutgoingHandshake:456(509ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:476(2267ns) {SwitchToState:178(516ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.580924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519247041469502:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23220480ns) {Configure:104(23216095ns) {ProcessConfigured:135(21318745ns) {StartInitialHandshake:157(21317266ns) {DropHandshakes:469(3725ns) {DropIncomingHandshake:441(160ns) DropOutgoingHandshake:456(84ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:476(2360ns) {SwitchToState:178(925ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.584580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691592:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.621904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519246636446336:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(64444222ns) {Configure:104(64439536ns) {ProcessConfigured:135(14906877ns) {StartInitialHandshake:157(14904071ns) {DropHandshakes:469(2167ns) {DropIncomingHandshake:441(263ns) DropOutgoingHandshake:456(468ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:476(4179ns) {SwitchToState:178(1815ns)}}}}}
2025-11-11T17:21:39.629774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519245900461081:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.580316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519244161318679:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.585124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519247041469501:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(57302191ns) {Configure:104(57297207ns) {ProcessConfigured:135(16218420ns) {StartInitialHandshake:157(16216150ns) {DropHandshakes:469(3297ns) {DropIncomingHandshake:441(873ns) DropOutgoingHandshake:456(577ns)} GenerateSessionVirtualId:988(449ns) PrepareNewSessionHandshake:476(2791ns) {SwitchToState:178(970ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.591655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519247616278963:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45297
2025-11-11T17:21:39.621984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519246636446335:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(64725100ns) {Configure:104(64721023ns) {ProcessConfigured:135(14382315ns) {StartInitialHandshake:157(14379740ns) {DropHandshakes:469(1229ns) {DropIncomingHandshake:441(391ns) DropOutgoingHandshake:456(212ns)} GenerateSessionVirtualId:988(1650ns) PrepareNewSessionHandshake:476(1744ns) {SwitchToState:178(445ns)}}}}}
2025-11-11T17:21:39.646046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519247041469500:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(20344555ns) {IncomingHandshake:261(4905697ns) {DropIncomingHandshake:441(743ns) PrepareNewSessionHandshake:476(2449ns) {SwitchToState:178(1067ns)} GenerateSessionVirtualId:988(755ns) IssueIncomingHandshakeReply:193(8875ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.597852Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519246806691590:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:39.613877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519246806691567:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.613947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:39.613993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519246806691567:2106] [node 5] ICP22 created new session: [6:7571519246806691593:2106]
2025-11-11T17:21:39.614066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519246806691593:2106] [node 5] ICS09 handshake done sender: [6:7571519246806691590:2122] self: [6:7571519246806691591:0] peer: [5:7571519244161318685:0] socket: 106 qp: -1
2025-11-11T17:21:39.635917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519246806691593:2106] [node 5] ICS10 traffic start
2025-11-11T17:21:39.635967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:39.658048Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519246636446338:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.642118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691594:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.663700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691597:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.663281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519246636446352:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46928
2025-11-11T17:21:39.677038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519246806691594:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.676648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519246636446353:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46928
2025-11-11T17:21:39.677330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519246806691505:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.677499Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519246636446353:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.689862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691600:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.690398Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519246806691594:2124] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:39.606961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519244161318617:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(91483869ns) {Configure:104(91478616ns) {ProcessConfigured:135(65998ns) {StartInitialHandshake:157(64069ns) {DropHandshakes:469(3341ns) {DropIncomingHandshake:441(689ns) DropOutgoingHandshake:456(618ns)} GenerateSessionVirtualId:988(1097ns) PrepareNewSessionHandshake:476(2156ns) {SwitchToState:178(650ns)}}}}}
2025-11-11T17:21:39.684222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519246636446254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.611773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519247616278961:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45297
2025-11-11T17:21:39.642811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519245900461086:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
2025-11-11T17:21:39.695784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519246636446352:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.715475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519246636446356:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46928
2025-11-11T17:21:39.701828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519246806691505:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.698196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519247041469519:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35146
2025-11-11T17:21:39.634842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278963:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.614583Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519244161318616:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(77382306ns) {Configure:104(77375782ns) {ProcessConfigured:135(28074173ns) {StartInitialHandshake:157(24186206ns) {DropHandshakes:469(1937ns) {DropIncomingHandshake:441(252ns) DropOutgoingHandshake:456(367ns)} GenerateSessionVirtualId:988(1466ns) PrepareNewSessionHandshake:476(1971ns) {SwitchToState:178(179ns)}}}}}
2025-11-11T17:21:39.722047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519246636446255:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.655506Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519245900461083:2131] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.709979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.724951Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519246636446342:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.637881Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278961:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.662723Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519245900461085:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.721841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519246806691505:2048] [node 1] ICP22 created new session: [6:7571519246806691601:2048]
2025-11-11T17:21:39.634410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318698:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:39.747779Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469519:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.724996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519246636446342:2129] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.643436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519247616278970:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45297
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.669988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519245900460994:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(157119983ns) {Configure:104(157112267ns) {ProcessConfigured:135(23347418ns) {StartInitialHandshake:157(23344389ns) {DropHandshakes:469(7080660ns) {DropIncomingHandshake:441(1312ns) DropOutgoingHandshake:456(486ns)} GenerateSessionVirtualId:988(1211ns) PrepareNewSessionHandshake:476(4562ns) {SwitchToState:178(2132ns)}}}}}
2025-11-11T17:21:39.748233Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469505:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.634469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318699:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:39.790594Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519246636446342:2129] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:39.645120Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278970:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.721901Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:39.727353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519245900461080:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.638222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519244161318680:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.727488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:39.727539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519245900461080:2109] [node 5] ICP22 created new session: [4:7571519245900461093:2109]
2025-11-11T17:21:39.727619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519245900461093:2109] [node 5] ICS09 handshake done sender: [4:7571519245900461083:2131] self: [4:7571519245900461082:0] peer: [5:7571519244161318695:0] socket: 103 qp: -1
2025-11-11T17:21:39.727666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519245900461093:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:39.727740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:39.733301Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519245900461086:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.733454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519245900460994:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.763496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519245900461075:2128] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(159354176ns) {Configure:104(159347335ns) {ProcessConfigured:135(123012ns) {StartInitialHandshake:157(120068ns) {DropHandshakes:469(3457ns) {DropIncomingHandshake:441(958ns) DropOutgoingHandshake:456(722ns)} GenerateSessionVirtualId:988(1762ns) PrepareNewSessionHandshake:476(8629ns) {SwitchToState:178(787ns)}}}}}
2025-11-11T17:21:39.799547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519245900461098:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
2025-11-11T17:21:39.836685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519245900461098:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.836848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519245900460992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.840842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519245900461100:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
2025-11-11T17:21:39.748269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519247041469505:2128] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.681996Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278957:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.735496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519246806691601:2048] [node 1] ICS09 handshake done sender: [6:7571519246806691594:2124] self: [6:7571519246806691598:0] peer: [1:7571519247616278956:0] socket: 116 qp: -1
2025-11-11T17:21:39.793664Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519246636446353:2135] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:39.646578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:39.763453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519247041469518:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35146
2025-11-11T17:21:39.797198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519246636446359:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46928
2025-11-11T17:21:39.682023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519247616278957:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.745548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519246806691601:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:39.658881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519244161318680:2110] [node 6] ICP22 created new session: [5:7571519244161318701:2110]
2025-11-11T17:21:39.846804Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519245900461086:2133] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.682923Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519247616278957:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:39.763609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:39.773351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519247041469521:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35146
2025-11-11T17:21:39.797269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519246636446360:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46928
2025-11-11T17:21:39.851837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519245900460994:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.790356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691599:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.798586Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519246806691597:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.823742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519246806691505:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(121360373ns) {HandleHandshakeStatus:329(121356669ns) {DropIncomingHandshake:441(917ns) DropOutgoingHandshake:456(15130ns) SwitchToState:178(889ns) ProcessPendingSessionEvents:495(10255688ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10210246ns max# 9802567ns}}}
2025-11-11T17:21:39.823812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519246806691505:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:39.824365Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519246806691599:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.843714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519246806691567:2106] [node 5] ICP35 event processing took too much time PendingConnection:278(84921877ns) {HandleHandshakeStatus:329(84918150ns) {DropIncomingHandshake:441(486ns) DropOutgoingHandshake:456(17224ns) SwitchToState:178(735ns) ProcessPendingSessionEvents:495(24588086ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24512996ns max# 16653677ns}}}
2025-11-11T17:21:39.844165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519246806691507:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.850505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519246806691506:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.851105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519246806691507:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(31138938ns) {IncomingHandshake:261(19733244ns) {DropIncomingHandshake:441(970ns) PrepareNewSessionHandshake:476(2180ns) {SwitchToState:178(955ns)} GenerateSessionVirtualId:988(934ns) IssueIncomingHandshakeReply:193(4381040ns) {SwitchToState:178(1911ns)}}}
2025-11-11T17:21:39.858738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691606:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.683250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519247616278953:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.774745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519247041469420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.797838Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519247041469505:2128] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:39.808546Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469518:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.808624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519247041469421:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.808705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:39.808758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519247041469421:2049] [node 3] ICP22 created new session: [2:7571519247041469523:2049]
2025-11-11T17:21:39.808806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519247041469523:2049] [node 3] ICS09 handshake done sender: [2:7571519247041469505:2128] self: [2:7571519247041469504:0] peer: [3:7571519246636446357:0] socket: 102 qp: -1
2025-11-11T17:21:39.808844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519247041469523:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:39.812689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:39.840533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519247041469527:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35146
2025-11-11T17:21:39.846393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469521:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.853731Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:39.855865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469514:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.855893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519247041469514:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.863609Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519247041469514:2131] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:39.873119Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469515:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.873161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519247041469515:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.674300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519244161318701:2110] [node 6] ICS09 handshake done sender: [5:7571519244161318686:2112] self: [5:7571519244161318685:0] peer: [6:7571519246806691591:0] socket: 105 qp: -1
2025-11-11T17:21:39.800909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519246636446344:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.856522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:39.677510Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519244161318698:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.683362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:39.901113Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519246806691599:2126] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:39.902141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519247041469502:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.873930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519245900460994:2050] [node 3] ICP22 created new session: [4:7571519245900461102:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.677719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519244161318701:2110] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.918078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519246806691506:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.902262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:39.800947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519246636446344:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.683449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519247616278953:2109] [node 6] ICP22 created new session: [1:7571519247616278974:2109]
2025-11-11T17:21:39.897839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519245900461102:2050] [node 3] ICS09 handshake done sender: [4:7571519245900461086:2133] self: [4:7571519245900461095:0] peer: [3:7571519246636446340:0] socket: 120 qp: -1
2025-11-11T17:21:39.684222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:39.811784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519246636446254:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.938829Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.903993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519245900461102:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:39.902315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519247041469502:2111] [node 6] ICP22 created new session: [2:7571519247041469531:2111]
2025-11-11T17:21:39.683516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519247616278974:2109] [node 6] ICS09 handshake done sender: [1:7571519247616278957:2111] self: [1:7571519247616278956:0] peer: [6:7571519246806691598:0] socket: 114 qp: -1
2025-11-11T17:21:39.906531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519245900461101:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
2025-11-11T17:21:39.908074Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:39.914146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519245900460992:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519245900461098:2136] held: no
2025-11-11T17:21:39.915266Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519245900461098:2136] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:39.925019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519245900460992:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.925109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:39.925166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519245900460992:2048] [node 1] ICP22 created new session: [4:7571519245900461105:2048]
2025-11-11T17:21:39.925215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519245900461105:2048] [node 1] ICS09 handshake done sender: [4:7571519245900461098:2136] self: [4:7571519245900461099:0] peer: [1:7571519247616278976:0] socket: 143 qp: -1
2025-11-11T17:21:39.925252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519245900461105:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:39.925287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:39.925842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519245900461104:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
2025-11-11T17:21:39.929892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:39.991273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519245900460994:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(138602556ns) {HandleHandshakeStatus:329(138599826ns) {DropIncomingHandshake:441(524ns) DropOutgoingHandshake:456(7935ns) SwitchToState:178(108ns) ProcessPendingSessionEvents:495(23573727ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23514677ns max# 23052763ns}}}
2025-11-11T17:21:39.994005Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519245900461104:2139] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.686300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318702:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:39.953181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:39.902363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519247041469531:2111] [node 6] ICS09 handshake done sender: [2:7571519247041469514:2131] self: [2:7571519247041469510:0] peer: [6:7571519246806691605:0] socket: 126 qp: -1
2025-11-11T17:21:39.811867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:39.683556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519247616278974:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:40.008955Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519245900461092:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.690855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519244161318615:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.953263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519246806691506:2049] [node 2] ICP22 created new session: [6:7571519246806691609:2049]
2025-11-11T17:21:40.009017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519245900461092:2135] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.811916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519246636446254:2048] [node 1] ICP22 created new session: [3:7571519246636446361:2048]
2025-11-11T17:21:39.953316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519246806691609:2049] [node 2] ICS09 handshake done sender: [6:7571519246806691599:2126] self: [6:7571519246806691605:0] peer: [2:7571519247041469510:0] socket: 131 qp: -1
2025-11-11T17:21:39.902403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519247041469531:2111] [node 6] ICS10 traffic start
2025-11-11T17:21:39.683615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:39.685726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519247616278951:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(158151956ns) {Configure:104(147373212ns) {ProcessConfigured:135(60364ns) {StartInitialHandshake:157(57218ns) {DropHandshakes:469(2471ns) {DropIncomingHandshake:441(512ns) DropOutgoingHandshake:456(523ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:476(1163ns) {SwitchToState:178(315ns)}}}}}
2025-11-11T17:21:39.698780Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278959:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.698822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519247616278959:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.709697Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278965:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.709758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519247616278965:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.721892Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519247616278959:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:39.725392Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519247616278965:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:39.751500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519247616278954:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.751593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519247616278890:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.751676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:39.751730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519247616278890:2049] [node 3] ICP22 created new session: [1:7571519247616278982:2049]
2025-11-11T17:21:39.751787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519247616278982:2049] [node 3] ICS09 handshake done sender: [1:7571519247616278959:2112] self: [1:7571519247616278958:0] peer: [3:7571519246636446355:0] socket: 113 qp: -1
2025-11-11T17:21:39.751829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519247616278982:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:39.751879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:39.760523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:39.760586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519247616278954:2108] [node 5] ICP22 created new session: [1:7571519247616278984:2108]
2025-11-11T17:21:39.760631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519247616278984:2108] [node 5] ICS09 handshake done sender: [1:7571519247616278965:2116] self: [1:7571519247616278964:0] peer: [5:7571519244161318704:0] socket: 115 qp: -1
2025-11-11T17:21:39.760675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519247616278984:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:39.760714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:39.800580Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278962:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.800622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519247616278962:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.837285Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519247616278977:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.837311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519247616278977:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.845813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519247616278890:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(93424367ns) {HandleHandshakeStatus:329(93421590ns) {DropIncomingHandshake:441(595ns) DropOutgoingHandshake:456(572ns) SwitchToState:178(456ns) ProcessPendingSessionEvents:495(92876687ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 85785642ns max# 20084902ns}}}
2025-11-11T17:21:39.846738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519247616278962:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:39.853659Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519247616278977:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:39.863335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519247616278889:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.874635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519247616278951:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.893351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:39.898720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519247616278889:2048] [node 2] ICP22 created new session: [1:7571519247616278989:2048]
2025-11-11T17:21:39.904562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519247616278989:2048] [node 2] ICS09 handshake done sender: [1:7571519247616278962:2114] self: [1:7571519247616278960:0] peer: [2:7571519247041469520:0] socket: 134 qp: -1
2025-11-11T17:21:39.908592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519247616278989:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:39.940704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:39.940763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519247616278951:2107] [node 4] ICP22 created new session: [1:7571519247616278990:2107]
2025-11-11T17:21:39.947650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:39.961531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519247616278990:2107] [node 4] ICS09 handshake done sender: [1:7571519247616278977:2118] self: [1:7571519247616278976:0] peer: [4:7571519245900461099:0] socket: 142 qp: -1
2025-11-11T17:21:39.972040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519247616278990:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:39.980813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:40.007925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519247616278951:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(143341197ns) {HandleHandshakeStatus:329(133757088ns) {DropIncomingHandshake:441(16781ns) DropOutgoingHandshake:456(674ns) SwitchToState:178(635ns) ProcessPendingSessionEvents:495(12951155ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12893949ns max# 12504782ns}}}
2025-11-11T17:21:40.011658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519245900460993:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.693831Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519244161318699:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.902443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:39.953356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519246806691609:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:39.811961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519246636446361:2048] [node 1] ICS09 handshake done sender: [3:7571519246636446353:2135] self: [3:7571519246636446355:0] peer: [1:7571519247616278958:0] socket: 128 qp: -1
2025-11-11T17:21:39.694088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519244161318617:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:40.048005Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519245900461092:2135] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:39.811989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519246636446361:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:40.066772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519247616278889:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(199083317ns) {HandleHandshakeStatus:329(199080073ns) {DropIncomingHandshake:441(19288ns) DropOutgoingHandshake:456(753ns) SwitchToState:178(792ns) ProcessPendingSessionEvents:495(75155010ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75029090ns max# 13165167ns}}}
2025-11-11T17:21:39.953436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:39.909634Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:40.067089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519245900461075:2128] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:40.067210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:39.812025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:40.067264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519245900461075:2128] [node 6] ICP22 created new session: [4:7571519250195428406:2128]
2025-11-11T17:21:39.829893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519246636446254:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17252256ns) {HandleHandshakeStatus:329(17249579ns) {DropIncomingHandshake:441(831ns) DropOutgoingHandshake:456(5433ns) SwitchToState:178(413ns) ProcessPendingSessionEvents:495(16864127ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16789523ns max# 12332291ns}}}
2025-11-11T17:21:40.067346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519250195428406:2128] [node 6] ICS09 handshake done sender: [4:7571519245900461092:2135] self: [4:7571519245900461091:0] peer: [6:7571519246806691611:0] socket: 127 qp: -1
2025-11-11T17:21:39.829968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519246636446336:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:40.067401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519250195428406:2128] [node 6] ICS10 traffic start
2025-11-11T17:21:39.830019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:40.067456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:39.830066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519246636446336:2108] [node 5] ICP22 created new session: [3:7571519246636446365:2108]
2025-11-11T17:21:39.830130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519246636446365:2108] [node 5] ICS09 handshake done sender: [3:7571519246636446342:2129] self: [3:7571519246636446339:0] peer: [5:7571519244161318705:0] socket: 117 qp: -1
2025-11-11T17:21:39.830174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519246636446365:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:39.830210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:39.832388Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519246636446352:2134] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:39.835259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519246636446255:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.835382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:39.835433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519246636446255:2049] [node 2] ICP22 created new session: [3:7571519246636446368:2049]
2025-11-11T17:21:39.955041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519246806691506:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(36132487ns) {HandleHandshakeStatus:329(36128416ns) {DropIncomingHandshake:441(7047ns) DropOutgoingHandshake:456(14151ns) SwitchToState:178(790ns) ProcessPendingSessionEvents:495(586963ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 525358ns max# 87145ns}}}
2025-11-11T17:21:39.835480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519246636446368:2049] [node 2] ICS09 handshake done sender: [3:7571519246636446352:2134] self: [3:7571519246636446357:0] peer: [2:7571519247041469504:0] socket: 118 qp: -1
2025-11-11T17:21:39.835517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519246636446368:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:39.955311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519246806691507:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519246806691597:2125] held: no
2025-11-11T17:21:39.835592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:39.957269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691608:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.844026Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519246636446359:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.959895Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:39.847663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519246636446344:2130] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:39.853688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519246636446367:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46928
2025-11-11T17:21:39.961185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519246806691506:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:39.853964Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:39.966077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519246806691608:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.982992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519246806691579:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:39.722265Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519244161318687:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:40.008617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519246806691607:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.751390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519244161318679:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:40.024945Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519246806691597:2125] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:39.751485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:40.043893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519246806691507:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:40.047749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519251101658908:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44301
2025-11-11T17:21:39.757027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318706:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:40.070898Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519246806691608:2130] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:39.757536Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17653) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:40.071396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:39.781043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519244161318679:2109] [node 4] ICP22 created new session: [5:7571519244161318707:2109]
2025-11-11T17:21:40.081828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519246806691507:2050] [node 3] ICP22 created new session: [6:7571519251101658909:2050]
2025-11-11T17:21:39.797554Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519244161318698:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:40.081891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519251101658909:2050] [node 3] ICS09 handshake done sender: [6:7571519246806691597:2125] self: [6:7571519246806691604:0] peer: [3:7571519246636446341:0] socket: 123 qp: -1
2025-11-11T17:21:39.798947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318708:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:40.081936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519251101658909:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:39.800278Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:40.081992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:39.810312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519244161318707:2109] [node 4] ICS09 handshake done sender: [5:7571519244161318687:2113] self: [5:7571519244161318695:0] peer: [4:7571519245900461082:0] socket: 104 qp: -1
2025-11-11T17:21:40.101687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519246806691579:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.810368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519244161318707:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:40.101788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:39.812664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:40.101838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519246806691579:2110] [node 4] ICP22 created new session: [6:7571519251101658911:2110]
2025-11-11T17:21:39.813428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519244161318615:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519244161318698:2117] held: no
2025-11-11T17:21:40.101896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519251101658911:2110] [node 4] ICS09 handshake done sender: [6:7571519246806691608:2130] self: [6:7571519246806691611:0] peer: [4:7571519245900461091:0] socket: 157 qp: -1
2025-11-11T17:21:39.814029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519244161318615:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:40.101950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519251101658911:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:39.814104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:40.102003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:39.814150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519244161318615:2048] [node 1] ICP22 created new session: [5:7571519244161318710:2048]
2025-11-11T17:21:39.930289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519247041469420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519247041469519:2134] held: no
2025-11-11T17:21:39.814194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519244161318710:2048] [node 1] ICS09 handshake done sender: [5:7571519244161318698:2117] self: [5:7571519244161318704:0] peer: [1:7571519247616278964:0] socket: 121 qp: -1
2025-11-11T17:21:39.814224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519244161318710:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:39.814256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:39.940402Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519247041469515:2132] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:39.815148Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519244161318699:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:39.964875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519247041469420:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(45419596ns) {HandleHandshakeStatus:395(45413279ns) {LogHandshakeFail:472(21093ns) {UpdateErrorStateLog:544(5480ns)} DropOutgoingHandshake:456(1664ns) ProcessPendingSessionEvents:495(33745276ns) {unordered DropSessionEvent:512 num# 100 sum# 33710806ns max# 18095395ns}}}
2025-11-11T17:21:39.818720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519244161318679:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(66495273ns) {HandleHandshakeStatus:329(66492337ns) {DropIncomingHandshake:441(1415ns) DropOutgoingHandshake:456(5504ns) SwitchToState:178(774ns) ProcessPendingSessionEvents:495(577485ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 519522ns max# 50295ns}}}
2025-11-11T17:21:39.819063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519244161318679:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:39.979990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519247041469501:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.822104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318712:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:39.980107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:39.980165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519247041469501:2108] [node 5] ICP22 created new session: [2:7571519247041469535:2108]
2025-11-11T17:21:39.841150Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519244161318712:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.980216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519247041469535:2108] [node 5] ICS09 handshake done sender: [2:7571519247041469515:2132] self: [2:7571519247041469512:0] peer: [5:7571519244161318716:0] socket: 145 qp: -1
2025-11-11T17:21:39.841254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519244161318617:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:39.980252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519247041469535:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:39.841334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:39.980288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:39.841383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519244161318617:2050] [node 3] ICP22 created new session: [5:7571519244161318714:2050]
2025-11-11T17:21:39.980501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519247041469529:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35146
2025-11-11T17:21:39.841439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519244161318714:2050] [node 3] ICS09 handshake done sender: [5:7571519244161318699:2118] self: [5:7571519244161318705:0] peer: [3:7571519246636446339:0] socket: 124 qp: -1
2025-11-11T17:21:40.008717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519247041469421:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(199222567ns) {HandleHandshakeStatus:329(199214033ns) {DropIncomingHandshake:441(741ns) DropOutgoingHandshake:456(868ns) SwitchToState:178(885ns) ProcessPendingSessionEvents:495(157196825ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 152879192ns max# 22242205ns}}}
2025-11-11T17:21:39.841478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519244161318714:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:39.841514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:40.009453Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519247041469519:2134] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:39.850827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519244161318616:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:40.033970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519247041469420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:40.044938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519247041469513:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:40.009237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519244161318717:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-11T17:21:40.044985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519247041469513:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:40.011739Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:40.055478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:40.017058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519244161318680:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(221336421ns) {HandleHandshakeStatus:329(221332916ns) {DropIncomingHandshake:441(8295722ns) DropOutgoingHandshake:456(869ns) SwitchToState:178(709ns) ProcessPendingSessionEvents:495(152578337ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 149302111ns max# 23689368ns}}}
2025-11-11T17:21:40.055560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519247041469420:2048] [node 1] ICP22 created new session: [2:7571519251336436833:2048]
2025-11-11T17:21:40.041887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519244161318616:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519244161318712:2122] held: no
2025-11-11T17:21:40.070708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519251336436833:2048] [node 1] ICS09 handshake done sender: [2:7571519247041469519:2134] self: [2:7571519247041469520:0] peer: [1:7571519247616278960:0] socket: 135 qp: -1
2025-11-11T17:21:40.078060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519251336436833:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:40.083162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:40.097908Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519247041469513:2130] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:39.870080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519246636446335:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:39.870190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:39.870239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519246636446335:2107] [node 4] ICP22 created new session: [3:7571519246636446371:2107]
2025-11-11T17:21:39.870286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519246636446371:2107] [node 4] ICS09 handshake done sender: [3:7571519246636446344:2130] self: [3:7571519246636446340:0] peer: [4:7571519245900461095:0] socket: 119 qp: -1
2025-11-11T17:21:39.870340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519246636446371:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:39.870383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:39.874395Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519246636446345:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:39.874433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519246636446345:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:39.898639Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519246636446345:2131] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:40.105259Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17653) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:40.109090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519250195428408:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62788
2025-11-11T17:21:40.122270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519247041469500:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:40.125752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519247041469420:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(83972351ns) {HandleHandshakeStatus:329(83969106ns) {DropIncomingHandshake:441(1906ns) DropOutgoingHandshake:456(333ns) SwitchToState:178(738ns) ProcessPendingSessionEvents:495(1630ns)}}
2025-11-11T17:21:39.933693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519246636446337:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:40.137742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:40.137829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519247041469500:2107] [node 4] ICP22 created new session: [2:7571519251336436836:2107]
2025-11-11T17:21:40.137894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519251336436836:2107] [node 4] ICS09 handshake done sender: [2:7571519247041469513:2130] self: [2:7571519247041469506:0] peer: [4:7571519250195428404:0] socket: 112 qp: -1
2025-11-11T17:21:40.137946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519251336436836:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:40.138047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:40.123789Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519245900461104:2139] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:40.109647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519246806691579:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:40.141864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519246806691507:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(82252602ns) {HandleHandshakeStatus:329(82249231ns) {DropIncomingHandshake:441(2482ns) DropOutgoingHandshake:456(405ns) SwitchToState:178(532ns) ProcessPendingSessionEvents:495(1306ns)}}
2025-11-11T17:21:39.933779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:39.933835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519246636446337:2114] [node 6] ICP22 created new session: [3:7571519246636446374:2114]
2025-11-11T17:21:39.933898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519246636446374:2114] [node 6] ICS09 handshake done sender: [3:7571519246636446345:2131] self: [3:7571519246636446341:0] peer: [6:7571519246806691604:0] socket: 122 qp: -1
2025-11-11T17:21:39.941688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519246636446374:2114] [node 6] ICS10 traffic start
2025-11-11T17:21:39.958071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:40.078278Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519246636446255:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(56844497ns) {HandleHandshakeStatus:329(56841797ns) {DropIncomingHandshake:441(837ns) DropOutgoingHandshake:456(4878ns) SwitchToState:178(469ns) ProcessPendingSessionEvents:495(56378760ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56289125ns max# 14235630ns}}}
2025-11-11T17:21:40.117987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519246636446337:2114] [node 6] ICP35 event processing took too much time PendingConnection:278(176604366ns) {HandleHandshakeStatus:329(176600711ns) {DropIncomingHandshake:441(30623ns) DropOutgoingHandshake:456(434ns) SwitchToState:178(984ns) ProcessPendingSessionEvents:495(126656358ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 121680544ns max# 20501400ns}}}
2025-11-11T17:21:40.149670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519245900460993:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:40.152164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:40.152236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519245900460993:2049] [node 2] ICP22 created new session: [4:7571519250195428409:2049]
2025-11-11T17:21:40.152283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519250195428409:2049] [node 2] ICS09 handshake done sender: [4:7571519245900461104:2139] self: [4:7571519250195428404:0] peer: [2:7571519247041469506:0] socket: 155 qp: -1
2025-11-11T17:21:40.152322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519250195428409:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:40.152411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:40.166141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519247041469500:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(16633104ns) {HandleHandshakeStatus:329(16628050ns) {DropIncomingHandshake:441(29284ns) DropOutgoingHandshake:456(552ns) SwitchToState:178(1408ns) ProcessPendingSessionEvents:495(667528ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 593127ns max# 54677ns}}}
2025-11-11T17:21:40.175685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519244161318616:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(61315431ns) {HandleHandshakeStatus:395(61312237ns) {LogHandshakeFail:472(17502ns) {UpdateErrorStateLog:544(5484ns)} DropOutgoingHandshake:456(2525ns) ProcessPendingSessionEvents:495(44222381ns) {unordered DropSessionEvent:512 num# 100 sum# 8047638ns max# 5647149ns}}}
2025-11-11T17:21:40.175831Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519244161318712:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:40.176466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519244161318616:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:40.176876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:40.176931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519244161318616:2049] [node 2] ICP22 created new session: [5:7571519248456286014:2049]
2025-11-11T17:21:40.177004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519248456286014:2049] [node 2] ICS09 handshake done sender: [5:7571519244161318712:2122] self: [5:7571519244161318716:0] peer: [2:7571519247041469512:0] socket: 98 qp: -1
2025-11-11T17:21:40.177039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519248456286014:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:40.177113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:40.176881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519245900461075:2128] [node 6] ICP35 event processing took too much time PendingConnection:278(101948695ns) {HandleHandshakeStatus:329(101944852ns) {DropIncomingHandshake:441(27195ns) DropOutgoingHandshake:456(748ns) SwitchToState:178(623ns) ProcessPendingSessionEvents:495(101314652ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 101181386ns max# 16857501ns}}}
2025-11-11T17:21:40.258057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:40.258299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519251336436833:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-11T17:21:40.258338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519247041469420:2048] [node 1] ICP30 unregister session Session# [2:7571519251336436833:2048] VirtualId# [2:7571519247041469520:0]
2025-11-11T17:21:40.258374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519251336436833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.258453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:40.258496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:40.261750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:40.261985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519246636446361:2048] [node 1] ICS07 socket disconnect 128 reason# EndOfStream
2025-11-11T17:21:40.262014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519246636446361:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.262113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:40.262153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519246636446361:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:40.264664Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:21:40.265706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519246636446254:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:40.265745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519246636446361:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:40.270753Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address
2025-11-11T17:21:40.271095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519246636446254:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:40.271166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519246636446361:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:40.271211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519246636446254:2048] [node 1] ICP30 unregister session Session# [3:7571519246636446361:2048] VirtualId# [3:7571519246636446355:0]
2025-11-11T17:21:40.271259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:40.271289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519246636446254:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.271304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.277759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:40.278074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519244161318710:2048] [node 1] ICS01 socket: 121 reason# EndOfStream
2025-11-11T17:21:40.278131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519244161318615:2048] [node 1] ICP30 unregister session Session# [5:7571519244161318710:2048] VirtualId# [5:7571519244161318704:0]
2025-11-11T17:21:40.278166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519244161318710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.278367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:40.278430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:40.278324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:40.278573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519245900461105:2048] [node 1] ICS01 socket: 143 reason# EndOfStream
2025-11-11T17:21:40.278607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519245900460992:2048] [node 1] ICP30 unregister session Session# [4:7571519245900461105:2048] VirtualId# [4:7571519245900461099:0]
2025-11-11T17:21:40.278634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519245900461105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.278719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:40.278750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:40.277756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:40.277936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519246806691601:2048] [node 1] ICS07 socket disconnect 116 reason# EndOfStream
2025-11-11T17:21:40.277963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519246806691601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.278028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:40.278059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519246806691601:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:40.278879Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address
2025-11-11T17:21:40.279129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519246806691505:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:40.279164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519246806691601:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:40.279187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519246806691505:2048] [node 1] ICP30 unregister session Session# [6:7571519246806691601:2048] VirtualId# [6:7571519246806691598:0]
2025-11-11T17:21:40.279262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:40.279282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519246806691505:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.279292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.283476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519246636446254:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:40.295160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519246806691505:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:40.361009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:40.361281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519246806691609:2049] [node 2] ICS07 socket disconnect 131 reason# EndOfStream
2025-11-11T17:21:40.361322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519246806691609:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.361406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:40.361450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519246806691609:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:40.361661Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:40.361190Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:40.361626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519250195428409:2049] [node 2] ICS01 socket: 155 reason# EndOfStream
2025-11-11T17:21:40.362091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519245900460993:2049] [node 2] ICP30 unregister session Session# [4:7571519250195428409:2049] VirtualId# [4:7571519250195428404:0]
2025-11-11T17:21:40.362134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519250195428409:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.362465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:40.362532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:40.364055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519248456286014:2049] [node 2] ICS01 socket: 98 reason# EndOfStream
2025-11-11T17:21:40.364185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:40.364417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519246636446368:2049] [node 2] ICS07 socket disconnect 118 reason# EndOfStream
2025-11-11T17:21:40.369379Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19081) Couldn't connect to any resolved address
2025-11-11T17:21:40.369439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519246636446368:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.369569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:40.370456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519246806691506:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19081) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:40.370526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519246806691609:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:40.369877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519246636446368:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:40.370564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519246806691506:2049] [node 2] ICP30 unregister session Session# [6:7571519246806691609:2049] VirtualId# [6:7571519246806691605:0]
2025-11-11T17:21:40.370616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:40.370671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519246806691506:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19081) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.370693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19081) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.364127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519244161318616:2049] [node 2] ICP30 unregister session Session# [5:7571519248456286014:2049] VirtualId# [5:7571519244161318716:0]
2025-11-11T17:21:40.364163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519248456286014:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.364256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:40.365576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:40.383677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519246806691506:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:40.433021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:40.433284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519245900461102:2050] [node 3] ICS07 socket disconnect 120 reason# EndOfStream
2025-11-11T17:21:40.433325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519245900461102:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.433423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:40.433465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519245900461102:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:40.434120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:40.434349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519244161318714:2050] [node 3] ICS07 socket disconnect 124 reason# EndOfStream
2025-11-11T17:21:40.434394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519244161318714:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.434509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:40.434239Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address
2025-11-11T17:21:40.434358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519245900460994:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:40.434424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519245900461102:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:40.434510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519245900460994:2050] [node 3] ICP30 unregister session Session# [4:7571519245900461102:2050] VirtualId# [4:7571519245900461095:0]
2025-11-11T17:21:40.434564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519244161318714:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:40.434918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:40.434969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519245900460994:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.434987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.435587Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address
2025-11-11T17:21:40.435701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519244161318617:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:40.435802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519244161318714:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:40.435828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519244161318617:2050] [node 3] ICP30 unregister session Session# [5:7571519244161318714:2050] VirtualId# [5:7571519244161318705:0]
2025-11-11T17:21:40.435942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:40.436129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519251101658909:2050] [node 3] ICS01 socket: 123 reason# EndOfStream
2025-11-11T17:21:40.436166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519246806691507:2050] [node 3] ICP30 unregister session Session# [6:7571519251101658909:2050] VirtualId# [6:7571519246806691604:0]
2025-11-11T17:21:40.443039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:40.443129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519244161318617:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.443149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.436195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519251101658909:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.436263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:40.436317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:40.445779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519245900460994:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:40.453446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519244161318617:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:40.522249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:40.522493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519251101658911:2110] [node 4] ICS01 socket: 157 reason# EndOfStream
2025-11-11T17:21:40.522544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519246806691579:2110] [node 4] ICP30 unregister session Session# [6:7571519251101658911:2110] VirtualId# [6:7571519246806691611:0]
2025-11-11T17:21:40.522585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519251101658911:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.522674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:40.522753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:40.525858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:40.526258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519244161318707:2109] [node 4] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-11T17:21:40.526321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519244161318707:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.526484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:40.526582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519244161318707:2109] [node 4] ICS15 start handshake
2025-11-11T17:21:40.533869Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17653) Couldn't connect to any resolved address
2025-11-11T17:21:40.534134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519244161318679:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17653) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:40.534221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519244161318707:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:40.534258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519244161318679:2109] [node 4] ICP30 unregister session Session# [5:7571519244161318707:2109] VirtualId# [5:7571519244161318695:0]
2025-11-11T17:21:40.534374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:40.534419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519244161318679:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.534434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:40.544578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519244161318679:2109] [node 4] ICP33 wake up from error state
2025-11-11T17:21:40.623714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:40.623975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519246806691593:2106] [node 5] ICS01 socket: 106 reason# EndOfStream
2025-11-11T17:21:40.624039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519246806691567:2106] [node 5] ICP30 unregister session Session# [6:7571519246806691593:2106] VirtualId# [6:7571519246806691591:0]
2025-11-11T17:21:40.624074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519246806691593:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:40.624147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:40.624205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:40.733852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519251336157722:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:40.733911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519251336157723:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:40.754337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519252233295794:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:40.754406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519252233295795:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:40.777032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519249476838535:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:40.777081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519249476838536:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:40.797060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519251814275489:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:40.797119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519251814275490:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:40.797154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519251814275491:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:40.817978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519249888403014:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:40.818048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519249888403015:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:40.818086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519249888403016:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:40.841282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519250865517530:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:40.841324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519250865517531:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:40.841353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519250865517532:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:40.955179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519250865517591:2105] [node 5] ICP01 ready to work
2025-11-11T17:21:40.995451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519252233295855:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.019821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519249888403075:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.101887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519251336157722:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(74557111ns) {Configure:104(74356073ns) {ProcessConfigured:135(34642235ns) {StartInitialHandshake:157(34637997ns) {DropHandshakes:469(3873ns) {DropIncomingHandshake:441(816ns) DropOutgoingHandshake:456(595ns)} GenerateSessionVirtualId:988(1233ns) PrepareNewSessionHandshake:476(4872ns) {SwitchToState:178(1618ns)}}}}}
2025-11-11T17:21:41.112267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519254183370383:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:41.111739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519253771805894:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:41.105989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519256109242849:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:41.112955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519255160484893:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:41.106321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519256109242850:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:41.113251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519255631125083:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:41.113350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519253771805895:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:41.111624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519252233295855:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(66426740ns) {Configure:104(66420469ns) {ProcessConfigured:135(43682884ns) {StartInitialHandshake:157(43678202ns) {DropHandshakes:469(3453ns) {DropIncomingHandshake:441(834ns) DropOutgoingHandshake:456(677ns)} GenerateSessionVirtualId:988(901ns) PrepareNewSessionHandshake:476(5655ns) {SwitchToState:178(2538ns)}}}}}
2025-11-11T17:21:41.112712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519256528263160:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:41.118735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519255631125084:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:41.118729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519254183370386:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:41.119089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519253771805896:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:41.119650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519256528263161:2111] [node 6] ICP01 ready to work
2025-11-11T17:21:41.131000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519255631125085:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:41.134355Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519249888403075:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.146084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519253771805910:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19217
2025-11-11T17:21:41.141350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519255631125086:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31579
2025-11-11T17:21:41.144089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125086:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.163862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519249888403016:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19903608ns) {Configure:104(19900088ns) {ProcessConfigured:135(56754ns) {StartInitialHandshake:157(54090ns) {DropHandshakes:469(3174ns) {DropIncomingHandshake:441(656ns) DropOutgoingHandshake:456(623ns)} GenerateSessionVirtualId:988(1004ns) PrepareNewSessionHandshake:476(2092ns) {SwitchToState:178(733ns)}}}}}
2025-11-11T17:21:41.156880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519256109242850:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23025848ns) {Configure:104(23020275ns) {ProcessConfigured:135(15203562ns) {StartInitialHandshake:157(15199817ns) {DropHandshakes:469(806ns) {DropIncomingHandshake:441(249ns) DropOutgoingHandshake:456(208ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(4629ns) {SwitchToState:178(2262ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.173572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519256109242861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23843
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.179533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519253771805911:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19217
2025-11-11T17:21:41.187142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519255631125099:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31579
2025-11-11T17:21:41.181863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519256528263162:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30752
2025-11-11T17:21:41.183516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519253771805912:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19217
2025-11-11T17:21:41.187205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519255631125100:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31579
2025-11-11T17:21:41.187693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805910:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.183875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519256528263163:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30752
2025-11-11T17:21:41.187316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519255631125101:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31579
2025-11-11T17:21:41.188112Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125100:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.194920Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519256109242861:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.194406Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125090:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.194449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519255631125090:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.187917Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263162:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.187841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519249476838535:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.188038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263163:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.188178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519256528263170:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30752
2025-11-11T17:21:41.207109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519254183370383:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(38824924ns) {Configure:104(38819060ns) {ProcessConfigured:135(7827541ns) {StartInitialHandshake:157(7825059ns) {DropHandshakes:469(3388ns) {DropIncomingHandshake:441(860ns) DropOutgoingHandshake:456(740ns)} GenerateSessionVirtualId:988(1058ns) PrepareNewSessionHandshake:476(3542ns) {SwitchToState:178(1270ns)}}}}}
2025-11-11T17:21:41.207109Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519255631125090:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:41.188240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519256528263171:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30752
2025-11-11T17:21:41.195092Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263171:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.219320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519252233295794:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.218880Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125099:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.218927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519251336157723:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.220486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519253771805914:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19217
2025-11-11T17:21:41.220182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519251814275491:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.220290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519256109242865:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23843
2025-11-11T17:21:41.221042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263170:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.220000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:41.220036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519251336157723:2049] [node 3] ICP22 created new session: [1:7571519255631125104:2049]
2025-11-11T17:21:41.224592Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263159:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.224632Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519256528263159:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.220105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519255631125104:2049] [node 3] ICS09 handshake done sender: [1:7571519255631125090:2113] self: [1:7571519255631125089:0] peer: [3:7571519253771805913:0] socket: 104 qp: -1
2025-11-11T17:21:41.223975Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519256109242865:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.227855Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519256528263159:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:41.224078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519251814275490:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.228283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519252233295855:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.228391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:41.220132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519255631125104:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:41.220174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:41.225636Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125101:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.225724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125082:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.225738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519255631125082:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.231857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519256109242868:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23843
2025-11-11T17:21:41.228441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519252233295855:2107] [node 4] ICP22 created new session: [2:7571519256528263178:2107]
2025-11-11T17:21:41.228484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519256528263178:2107] [node 4] ICS09 handshake done sender: [2:7571519256528263159:2112] self: [2:7571519256528263158:0] peer: [4:7571519256109242867:0] socket: 123 qp: -1
2025-11-11T17:21:41.228524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519256528263178:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:41.228558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:41.232418Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519255631125082:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:41.238936Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8312) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.233894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519251336157722:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.233938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:41.233964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519251336157722:2048] [node 2] ICP22 created new session: [1:7571519255631125110:2048]
2025-11-11T17:21:41.234034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519255631125110:2048] [node 2] ICS09 handshake done sender: [1:7571519255631125082:2110] self: [1:7571519251336157785:0] peer: [2:7571519256528263174:0] socket: 102 qp: -1
2025-11-11T17:21:41.239086Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805912:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.239797Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.243670Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.234085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519255631125110:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:41.234123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:41.244755Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805902:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.244795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519253771805902:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.245462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519249476838535:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519253771805910:2115] held: no
2025-11-11T17:21:41.243775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519254183370386:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19887938ns) {Configure:104(19883089ns) {ProcessConfigured:135(360157ns) {StartInitialHandshake:157(356332ns) {DropHandshakes:469(3596ns) {DropIncomingHandshake:441(624ns) DropOutgoingHandshake:456(685ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:476(3439ns) {SwitchToState:178(1338ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.250492Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519256109242865:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.253380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484904:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
2025-11-11T17:21:41.251280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519252233295794:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519256528263163:2114] held: no
2025-11-11T17:21:41.252951Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519253771805910:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.254689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484905:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
2025-11-11T17:21:41.253893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519251814275490:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8312) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519256109242865:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.256008Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519255160484904:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.256134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519250865517530:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.256640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519251814275490:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.252224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519256528263180:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30752
2025-11-11T17:21:41.255936Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519256528263163:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:41.256738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:41.262692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519252233295794:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.262801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:41.262853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519252233295794:2048] [node 1] ICP22 created new session: [2:7571519256528263183:2048]
2025-11-11T17:21:41.262894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519256528263183:2048] [node 1] ICS09 handshake done sender: [2:7571519256528263163:2114] self: [2:7571519256528263174:0] peer: [1:7571519251336157785:0] socket: 107 qp: -1
2025-11-11T17:21:41.262931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519256528263183:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:41.262982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:41.256787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519251814275490:2049] [node 2] ICP22 created new session: [4:7571519256109242870:2049]
2025-11-11T17:21:41.256834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519256109242870:2049] [node 2] ICS09 handshake done sender: [4:7571519256109242865:2117] self: [4:7571519256109242867:0] peer: [2:7571519256528263158:0] socket: 124 qp: -1
2025-11-11T17:21:41.256869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519256109242870:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:41.256910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:41.258715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519256109242869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23843
2025-11-11T17:21:41.261580Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125094:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.266108Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519256109242869:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.261626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519255631125094:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.268194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519249476838535:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.269127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519251814275489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.269234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519256109242873:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23843
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.270214Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519255631125094:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.268739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519255160484905:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.269283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519250865517531:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.270628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484908:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.269858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370395:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.270763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370397:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.270889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370398:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.274065Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519254183370397:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.271428Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263165:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.270912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:41.280732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519254183370383:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.270978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519249476838535:2048] [node 1] ICP22 created new session: [3:7571519253771805916:2048]
2025-11-11T17:21:41.271077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519253771805916:2048] [node 1] ICS09 handshake done sender: [3:7571519253771805910:2115] self: [3:7571519253771805913:0] peer: [1:7571519255631125089:0] socket: 105 qp: -1
2025-11-11T17:21:41.271466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519256528263165:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.271122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519253771805916:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.287295Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519256109242858:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.293728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125092:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.271196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:41.293778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519255631125092:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.272369Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805911:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.294998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519255631125085:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.272426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519249476838536:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.295054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:41.273109Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519253771805902:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:41.295110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519255631125085:2109] [node 6] ICP22 created new session: [1:7571519255631125115:2109]
2025-11-11T17:21:41.283133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519253771805894:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.295191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519255631125115:2109] [node 6] ICS09 handshake done sender: [1:7571519255631125094:2115] self: [1:7571519255631125093:0] peer: [6:7571519255160484906:0] socket: 98 qp: -1
2025-11-11T17:21:41.287341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519256109242858:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.287692Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519255160484904:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:41.285892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:41.304333Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519254183370398:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.304963Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519254183370395:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.295233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519255631125115:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:41.300312Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263156:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.301850Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.288495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519253771805894:2107] [node 4] ICP22 created new session: [3:7571519253771805921:2107]
2025-11-11T17:21:41.314571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519250865517530:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.314630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:41.314677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519250865517530:2048] [node 1] ICP22 created new session: [6:7571519255160484909:2048]
2025-11-11T17:21:41.295267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:41.300355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519256528263156:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.296964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519255631125114:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31579
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.315674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370403:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.315763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519249888403015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.315838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519249888403014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.314719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519255160484909:2048] [node 1] ICS09 handshake done sender: [6:7571519255160484904:2115] self: [6:7571519255160484906:0] peer: [1:7571519255631125093:0] socket: 122 qp: -1
2025-11-11T17:21:41.302909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519253771805921:2107] [node 4] ICS09 handshake done sender: [3:7571519253771805902:2112] self: [3:7571519253771805901:0] peer: [4:7571519256109242866:0] socket: 100 qp: -1
2025-11-11T17:21:41.309771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519256528263185:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30752
2025-11-11T17:21:41.310013Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519256528263165:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:41.316496Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519256528263156:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:41.302882Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519255631125092:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:41.313995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519255631125083:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.314075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:41.315240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519255631125083:2107] [node 4] ICP22 created new session: [1:7571519255631125118:2107]
2025-11-11T17:21:41.315301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519255631125118:2107] [node 4] ICS09 handshake done sender: [1:7571519255631125092:2114] self: [1:7571519255631125091:0] peer: [4:7571519256109242874:0] socket: 129 qp: -1
2025-11-11T17:21:41.316330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519255631125118:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:41.316382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:41.318433Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125114:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.304143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519253771805921:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:41.317069Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519256109242861:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:41.315139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519255160484909:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:41.320689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519256528263161:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.316821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:41.304205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:41.319706Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.323464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519251814275491:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.323558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:41.323607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519251814275491:2050] [node 3] ICP22 created new session: [4:7571519256109242877:2050]
2025-11-11T17:21:41.323659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519256109242877:2050] [node 3] ICS09 handshake done sender: [4:7571519256109242861:2116] self: [4:7571519256109242866:0] peer: [3:7571519253771805901:0] socket: 108 qp: -1
2025-11-11T17:21:41.323696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519256109242877:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:41.323728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:41.324503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519251814275491:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:41.320798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:41.312159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519253771805894:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(27578236ns) {HandleHandshakeStatus:329(27572064ns) {DropIncomingHandshake:441(969ns) DropOutgoingHandshake:456(788ns) SwitchToState:178(582ns) ProcessPendingSessionEvents:495(560232ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 491295ns max# 61475ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.318017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519250865517530:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:41.325437Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519255631125088:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.320853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519256528263161:2111] [node 6] ICP22 created new session: [2:7571519256528263188:2111]
2025-11-11T17:21:41.325430Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519254183370403:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.327740Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.312546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519253771805894:2107] [node 4] ICP18 handshake (actor [3:7571519253771805911:2116]) is too old
2025-11-11T17:21:41.320906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519256528263188:2111] [node 6] ICS09 handshake done sender: [2:7571519256528263165:2115] self: [2:7571519256528263164:0] peer: [6:7571519255160484907:0] socket: 115 qp: -1
2025-11-11T17:21:41.320943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519256528263188:2111] [node 6] ICS10 traffic start
2025-11-11T17:21:41.320975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:41.332596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370406:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.334494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519251814275489:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519256109242869:2119] held: no
2025-11-11T17:21:41.325476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519255631125088:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.332689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519249888403016:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.336184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484911:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.336297Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263185:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.337109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519256109242875:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23843
2025-11-11T17:21:41.337870Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519256109242858:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:41.336435Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519256528263167:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.336461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519256528263167:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.313219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519253771805920:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19217
2025-11-11T17:21:41.315793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519253771805923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19217
2025-11-11T17:21:41.319449Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805920:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.326839Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519253771805912:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:41.338161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519249476838536:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.338212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:41.338269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519249476838536:2049] [node 2] ICP22 created new session: [3:7571519253771805926:2049]
2025-11-11T17:21:41.338323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519253771805926:2049] [node 2] ICS09 handshake done sender: [3:7571519253771805912:2117] self: [3:7571519253771805918:0] peer: [2:7571519256528263155:0] socket: 120 qp: -1
2025-11-11T17:21:41.338356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519253771805926:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:41.340115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:41.340443Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519255631125088:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:41.338073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484912:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
2025-11-11T17:21:41.341817Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519254183370397:2126] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:41.340703Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519255160484905:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:41.344166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805904:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.340851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519252233295795:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.342084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519254183370383:2109] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.344215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519253771805904:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.342180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:41.342222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519254183370383:2109] [node 4] ICP22 created new session: [5:7571519254183370409:2109]
2025-11-11T17:21:41.348510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519255631125084:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.348574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:41.342292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519254183370409:2109] [node 4] ICS09 handshake done sender: [5:7571519254183370397:2126] self: [5:7571519254183370402:0] peer: [4:7571519256109242857:0] socket: 135 qp: -1
2025-11-11T17:21:41.348614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519255631125084:2108] [node 5] ICP22 created new session: [1:7571519255631125122:2108]
2025-11-11T17:21:41.347664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519250865517531:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.349823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:41.349913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519256109242849:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.351055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519253771805928:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19217
2025-11-11T17:21:41.351198Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519253771805904:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:41.351480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519253771805895:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.351531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:41.351576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519253771805895:2108] [node 5] ICP22 created new session: [3:7571519253771805930:2108]
2025-11-11T17:21:41.351620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519253771805930:2108] [node 5] ICS09 handshake done sender: [3:7571519253771805904:2113] self: [3:7571519253771805903:0] peer: [5:7571519254183370407:0] socket: 144 qp: -1
2025-11-11T17:21:41.351651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519253771805930:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:41.350433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:41.348660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519255631125122:2108] [node 5] ICS09 handshake done sender: [1:7571519255631125088:2112] self: [1:7571519255631125087:0] peer: [5:7571519254183370405:0] socket: 103 qp: -1
2025-11-11T17:21:41.351715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:41.350657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:41.342324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519254183370409:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:41.350100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519252233295795:2049] [node 3] ICP22 created new session: [2:7571519256528263190:2049]
2025-11-11T17:21:41.348696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519255631125122:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:41.342350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:41.347399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370408:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.348282Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519254183370395:2125] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:41.355568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519249888403014:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.355656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:41.355704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519249888403014:2048] [node 1] ICP22 created new session: [5:7571519254183370411:2048]
2025-11-11T17:21:41.355747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519254183370411:2048] [node 1] ICS09 handshake done sender: [5:7571519254183370395:2125] self: [5:7571519254183370405:0] peer: [1:7571519255631125087:0] socket: 110 qp: -1
2025-11-11T17:21:41.355785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519254183370411:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:41.355844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:41.350632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519256528263190:2049] [node 3] ICS09 handshake done sender: [2:7571519256528263156:2110] self: [2:7571519256528263155:0] peer: [3:7571519253771805918:0] socket: 116 qp: -1
2025-11-11T17:21:41.358517Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.350489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519250865517531:2049] [node 2] ICP22 created new session: [6:7571519255160484914:2049]
2025-11-11T17:21:41.353921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519256109242849:2108] [node 5] ICP22 created new session: [4:7571519256109242880:2108]
2025-11-11T17:21:41.348727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:41.350528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519255160484914:2049] [node 2] ICS09 handshake done sender: [6:7571519255160484905:2116] self: [6:7571519255160484907:0] peer: [2:7571519256528263164:0] socket: 131 qp: -1
2025-11-11T17:21:41.352092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519256528263190:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:41.356298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519256109242880:2108] [node 5] ICS09 handshake done sender: [4:7571519256109242858:2114] self: [4:7571519256109242857:0] peer: [5:7571519254183370402:0] socket: 132 qp: -1
2025-11-11T17:21:41.357272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519256109242880:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:41.363841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:41.365303Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.350668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519255160484914:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:41.352123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:41.362802Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519255160484912:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.352338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:41.370042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519256109242849:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(18834151ns) {HandleHandshakeStatus:329(18831462ns) {DropIncomingHandshake:441(179ns) DropOutgoingHandshake:456(291ns) SwitchToState:178(153ns) ProcessPendingSessionEvents:495(1041757ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 976270ns max# 545230ns}}}
2025-11-11T17:21:41.370332Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519256109242869:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:41.374172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519249888403014:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.374294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519250865517532:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.374855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484916:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
2025-11-11T17:21:41.375233Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805928:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.375402Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519253771805906:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.375432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519253771805906:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.381665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519249888403016:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519254183370403:2128] held: no
2025-11-11T17:21:41.383588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370413:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.384188Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519254183370403:2128] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:41.375313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519251814275489:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.375416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:41.375460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519251814275489:2048] [node 1] ICP22 created new session: [4:7571519256109242882:2048]
2025-11-11T17:21:41.375507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519256109242882:2048] [node 1] ICS09 handshake done sender: [4:7571519256109242869:2119] self: [4:7571519256109242874:0] peer: [1:7571519255631125091:0] socket: 117 qp: -1
2025-11-11T17:21:41.375546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519256109242882:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:41.375812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:41.380996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519256109242884:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23843
2025-11-11T17:21:41.387466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519249888403016:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.387472Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519253771805906:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:41.387854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:41.387917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519249888403016:2050] [node 3] ICP22 created new session: [5:7571519254183370415:2050]
2025-11-11T17:21:41.387999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519254183370415:2050] [node 3] ICS09 handshake done sender: [5:7571519254183370403:2128] self: [5:7571519254183370407:0] peer: [3:7571519253771805903:0] socket: 146 qp: -1
2025-11-11T17:21:41.388039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519254183370415:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:41.388121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:41.391321Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519256528263167:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.392398Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519255160484916:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.392533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484919:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
2025-11-11T17:21:41.392418Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519256109242884:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.392741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484920:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.394180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519253771805896:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.394296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:41.393641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519255160484893:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.393705Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519255160484912:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:41.395272Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519255160484919:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.395324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519250865517532:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.395410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:41.395458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519250865517532:2050] [node 3] ICP22 created new session: [6:7571519255160484922:2050]
2025-11-11T17:21:41.395498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519255160484922:2050] [node 3] ICS09 handshake done sender: [6:7571519255160484912:2119] self: [6:7571519255160484917:0] peer: [3:7571519253771805905:0] socket: 152 qp: -1
2025-11-11T17:21:41.395532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519255160484922:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:41.395570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:41.394357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519253771805896:2109] [node 6] ICP22 created new session: [3:7571519253771805934:2109]
2025-11-11T17:21:41.394407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519253771805934:2109] [node 6] ICS09 handshake done sender: [3:7571519253771805906:2114] self: [3:7571519253771805905:0] peer: [6:7571519255160484917:0] socket: 112 qp: -1
2025-11-11T17:21:41.394441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519253771805934:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:41.394479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:41.394823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519254183370417:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-11-11T17:21:41.395218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519256528263160:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.396082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519256109242860:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.395383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:41.399142Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519254183370398:2127] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:41.399784Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8312) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.396114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519256109242860:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.395431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519256528263160:2109] [node 5] ICP22 created new session: [2:7571519256528263194:2109]
2025-11-11T17:21:41.395465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519256528263194:2109] [node 5] ICS09 handshake done sender: [2:7571519256528263167:2116] self: [2:7571519256528263166:0] peer: [5:7571519254183370404:0] socket: 134 qp: -1
2025-11-11T17:21:41.395493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519256528263194:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:41.395550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:41.397416Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519256109242860:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:41.399186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519256109242850:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.399554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:41.400406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519256109242850:2110] [node 6] ICP22 created new session: [4:7571519256109242887:2110]
2025-11-11T17:21:41.400467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519256109242887:2110] [node 6] ICS09 handshake done sender: [4:7571519256109242860:2115] self: [4:7571519256109242859:0] peer: [6:7571519255160484921:0] socket: 140 qp: -1
2025-11-11T17:21:41.400500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519256109242887:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:41.400549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:41.405016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519249888403015:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.405147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:41.405207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519249888403015:2049] [node 2] ICP22 created new session: [5:7571519254183370418:2049]
2025-11-11T17:21:41.405264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519254183370418:2049] [node 2] ICS09 handshake done sender: [5:7571519254183370398:2127] self: [5:7571519254183370404:0] peer: [2:7571519256528263166:0] socket: 136 qp: -1
2025-11-11T17:21:41.405310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519254183370418:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:41.405388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:41.406146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519249888403015:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:41.409778Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9053) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.411186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519250865517591:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:41.411275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519255160484893:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9053) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519255160484916:2120] held: no
2025-11-11T17:21:41.412435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484924:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
2025-11-11T17:21:41.412827Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519254183370394:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:41.412863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519254183370394:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:41.413999Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519254183370394:2124] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:41.414264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519254183370386:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:41.414365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:41.414418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519254183370386:2110] [node 6] ICP22 created new session: [5:7571519254183370421:2110]
2025-11-11T17:21:41.414534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519254183370421:2110] [node 6] ICS09 handshake done sender: [5:7571519254183370394:2124] self: [5:7571519254183370393:0] peer: [6:7571519255160484925:0] socket: 143 qp: -1
2025-11-11T17:21:41.414584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519254183370421:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:41.414653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:41.414828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519255160484916:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:41.415191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519255160484893:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.415260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:41.415302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519255160484893:2110] [node 4] ICP22 created new session: [6:7571519255160484927:2110]
2025-11-11T17:21:41.415339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519255160484927:2110] [node 4] ICS09 handshake done sender: [6:7571519255160484916:2120] self: [6:7571519255160484921:0] peer: [4:7571519256109242859:0] socket: 106 qp: -1
2025-11-11T17:21:41.415379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519255160484927:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:41.415407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:41.416123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519255160484926:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:268
2025-11-11T17:21:41.417219Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:41.418989Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519255160484919:2121] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:41.419303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519250865517591:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519255160484919:2121] held: no
2025-11-11T17:21:41.419899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519250865517591:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:41.419982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:41.420026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519250865517591:2105] [node 5] ICP22 created new session: [6:7571519255160484929:2105]
2025-11-11T17:21:41.420073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519255160484929:2105] [node 5] ICS09 handshake done sender: [6:7571519255160484919:2121] self: [6:7571519255160484925:0] peer: [5:7571519254183370393:0] socket: 157 qp: -1
2025-11-11T17:21:41.420105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519255160484929:2105] [node 5] ICS10 traffic start
2025-11-11T17:21:41.420143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:41.449473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:41.449711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519256528263183:2048] [node 1] ICS01 socket: 107 reason# EndOfStream
2025-11-11T17:21:41.449754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519252233295794:2048] [node 1] ICP30 unregister session Session# [2:7571519256528263183:2048] VirtualId# [2:7571519256528263174:0]
2025-11-11T17:21:41.449787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519256528263183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.449868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:41.449908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:41.451646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:41.451921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519253771805916:2048] [node 1] ICS01 socket: 105 reason# EndOfStream
2025-11-11T17:21:41.451960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519249476838535:2048] [node 1] ICP30 unregister session Session# [3:7571519253771805916:2048] VirtualId# [3:7571519253771805913:0]
2025-11-11T17:21:41.451996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519253771805916:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.452141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:41.452212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:41.453131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:41.453316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519256109242882:2048] [node 1] ICS01 socket: 117 reason# EndOfStream
2025-11-11T17:21:41.453356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519251814275489:2048] [node 1] ICP30 unregister session Session# [4:7571519256109242882:2048] VirtualId# [4:7571519256109242874:0]
2025-11-11T17:21:41.453385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519256109242882:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.453455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:41.453492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:41.456317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:41.456461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519254183370411:2048] [node 1] ICS07 socket disconnect 110 reason# EndOfStream
2025-11-11T17:21:41.456479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519254183370411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.456537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:41.456564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519254183370411:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:41.457106Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address
2025-11-11T17:21:41.457326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519249888403014:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:41.457859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:41.457360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519254183370411:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:41.457385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519249888403014:2048] [node 1] ICP30 unregister session Session# [5:7571519254183370411:2048] VirtualId# [5:7571519254183370405:0]
2025-11-11T17:21:41.457412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:41.457431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519249888403014:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.457440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.457972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519255160484909:2048] [node 1] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-11T17:21:41.462305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519255160484909:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.462425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:41.462468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519255160484909:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:41.467139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519249888403014:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:41.467592Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address
2025-11-11T17:21:41.467963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519250865517530:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:41.468018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519255160484909:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:41.468060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519250865517530:2048] [node 1] ICP30 unregister session Session# [6:7571519255160484909:2048] VirtualId# [6:7571519255160484906:0]
2025-11-11T17:21:41.468123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:41.468159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519250865517530:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.468175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.478543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519250865517530:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:41.509556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:41.509880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519253771805926:2049] [node 2] ICS01 socket: 120 reason# EndOfStream
2025-11-11T17:21:41.509921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519249476838536:2049] [node 2] ICP30 unregister session Session# [3:7571519253771805926:2049] VirtualId# [3:7571519253771805918:0]
2025-11-11T17:21:41.509963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519253771805926:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.510086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:41.510134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:41.512292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:41.512527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519256109242870:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-11T17:21:41.512580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519251814275490:2049] [node 2] ICP30 unregister session Session# [4:7571519256109242870:2049] VirtualId# [4:7571519256109242867:0]
2025-11-11T17:21:41.512615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519256109242870:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.512690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:41.512734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:41.514833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:41.515026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519255160484914:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:21:41.515073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519250865517531:2049] [node 2] ICP30 unregister session Session# [6:7571519255160484914:2049] VirtualId# [6:7571519255160484907:0]
2025-11-11T17:21:41.515105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519255160484914:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.514369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:41.515172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:41.514603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519254183370418:2049] [node 2] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-11T17:21:41.514633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519254183370418:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.514745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:41.514813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519254183370418:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:41.520025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:41.529759Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8312) Couldn't connect to any resolved address
2025-11-11T17:21:41.530097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519249888403015:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8312) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:41.530191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519254183370418:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:41.530354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519249888403015:2049] [node 2] ICP30 unregister session Session# [5:7571519254183370418:2049] VirtualId# [5:7571519254183370404:0]
2025-11-11T17:21:41.545202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:41.545279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519249888403015:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8312) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.545298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8312) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.555804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519249888403015:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:41.572016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:41.572278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519256109242877:2050] [node 3] ICS07 socket disconnect 108 reason# EndOfStream
2025-11-11T17:21:41.572306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519256109242877:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.572403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:41.572445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519256109242877:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:41.572022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:41.572106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:41.572335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519255160484922:2050] [node 3] ICS07 socket disconnect 152 reason# EndOfStream
2025-11-11T17:21:41.572361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519255160484922:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.572437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:41.572481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519255160484922:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:41.573671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519254183370415:2050] [node 3] ICS01 socket: 146 reason# EndOfStream
2025-11-11T17:21:41.573725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519249888403016:2050] [node 3] ICP30 unregister session Session# [5:7571519254183370415:2050] VirtualId# [5:7571519254183370407:0]
2025-11-11T17:21:41.573759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519254183370415:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.573832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:41.573873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:41.575426Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address
2025-11-11T17:21:41.575658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519251814275491:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:41.575717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519256109242877:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:41.575754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519251814275491:2050] [node 3] ICP30 unregister session Session# [4:7571519256109242877:2050] VirtualId# [4:7571519256109242866:0]
2025-11-11T17:21:41.575803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:41.575845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519251814275491:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.574742Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address
2025-11-11T17:21:41.575866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.575008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519250865517532:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:41.575057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519255160484922:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:41.575090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519250865517532:2050] [node 3] ICP30 unregister session Session# [6:7571519255160484922:2050] VirtualId# [6:7571519255160484917:0]
2025-11-11T17:21:41.575218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:41.575260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519250865517532:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.575275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:41.589982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519250865517532:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:41.590381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519251814275491:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:41.620065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:41.620333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519254183370409:2109] [node 4] ICS01 socket: 135 reason# EndOfStream
2025-11-11T17:21:41.620377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519254183370383:2109] [node 4] ICP30 unregister session Session# [5:7571519254183370409:2109] VirtualId# [5:7571519254183370402:0]
2025-11-11T17:21:41.620412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519254183370409:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.620494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:41.620539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:41.621115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:41.625796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519255160484927:2110] [node 4] ICS01 socket: 106 reason# EndOfStream
2025-11-11T17:21:41.625843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519255160484893:2110] [node 4] ICP30 unregister session Session# [6:7571519255160484927:2110] VirtualId# [6:7571519255160484921:0]
2025-11-11T17:21:41.627450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519255160484927:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.627576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:41.627621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:41.648816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:41.649043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519255160484929:2105] [node 5] ICS01 socket: 157 reason# EndOfStream
2025-11-11T17:21:41.649086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519250865517591:2105] [node 5] ICP30 unregister session Session# [6:7571519255160484929:2105] VirtualId# [6:7571519255160484925:0]
2025-11-11T17:21:41.649123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519255160484929:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:41.649231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:41.649277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:41.749033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519254846548922:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:41.749077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519254846548923:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:41.768241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519254804267137:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:41.768287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519254804267138:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:41.787163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519253053862736:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:41.787190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519253053862737:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:41.806526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519255205468338:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:41.806564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519255205468339:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:41.807100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519255205468337:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:41.832402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519254845524035:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:41.832478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519254845524036:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:41.832599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519254845524037:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:41.844845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519256127638827:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:41.844880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519256127638828:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:41.844926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519256127638829:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:41.971912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519256127638890:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:42.069774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519257348830112:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:42.069663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519254846548923:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18005543ns) {Configure:104(18002706ns) {ProcessConfigured:135(16894519ns) {StartInitialHandshake:157(16892608ns) {DropHandshakes:469(1809ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(182ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:476(4695ns) {SwitchToState:178(1508ns)}}}}}
2025-11-11T17:21:42.073987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519259500435696:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:42.070028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519259099234497:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:42.086076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519259140491395:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:42.086086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519259140491396:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:42.087569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519259141516283:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:42.088650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519257348830113:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:42.070293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519259099234498:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:42.073545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519259099234496:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:42.085735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519259500435697:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:42.089712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519256127638890:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(94462123ns) {Configure:104(94456317ns) {ProcessConfigured:135(57883610ns) {StartInitialHandshake:157(57880702ns) {DropHandshakes:469(2955ns) {DropIncomingHandshake:441(618ns) DropOutgoingHandshake:456(568ns)} GenerateSessionVirtualId:988(1084ns) PrepareNewSessionHandshake:476(5779ns) {SwitchToState:178(2730ns)}}}}}
2025-11-11T17:21:42.091787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519257348830114:2117] [node 6] ICP01 ready to work
2025-11-11T17:21:42.089983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519260422606190:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:42.089811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519259141516284:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:42.099185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519254846548922:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(118181003ns) {Configure:104(118176537ns) {ProcessConfigured:135(3863710ns) {StartInitialHandshake:157(3861733ns) {DropHandshakes:469(2572ns) {DropIncomingHandshake:441(509ns) DropOutgoingHandshake:456(805ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:476(2863ns) {SwitchToState:178(1022ns)}}}}}
2025-11-11T17:21:42.105410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519254845524035:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16664051ns) {Configure:104(16657575ns) {ProcessConfigured:135(34041ns) {StartInitialHandshake:157(31916ns) {DropHandshakes:469(479ns) {DropIncomingHandshake:441(190ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(596ns) {SwitchToState:178(99ns)}}}}}
2025-11-11T17:21:42.099330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519259141516287:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:42.121126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519255205468339:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18381693ns) {Configure:104(18379182ns) {ProcessConfigured:135(4751898ns) {StartInitialHandshake:157(4750565ns) {DropHandshakes:469(3752ns) {DropIncomingHandshake:441(749ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(729ns) PrepareNewSessionHandshake:476(2444ns) {SwitchToState:178(520ns)}}}}}
2025-11-11T17:21:42.121298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519259099234498:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(35235640ns) {Configure:104(35226843ns) {ProcessConfigured:135(54797ns) {StartInitialHandshake:157(52890ns) {DropHandshakes:469(2703ns) {DropIncomingHandshake:441(626ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(748ns) PrepareNewSessionHandshake:476(1922ns) {SwitchToState:178(788ns)}}}}}
2025-11-11T17:21:42.143680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519256127638827:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(30718686ns) {Configure:104(30714405ns) {ProcessConfigured:135(15525641ns) {StartInitialHandshake:157(15523306ns) {DropHandshakes:469(2445ns) {DropIncomingHandshake:441(375ns) DropOutgoingHandshake:456(209ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:476(3079ns) {SwitchToState:178(1157ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.151776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519257348830126:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2646
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.169949Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519257348830118:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.169986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519257348830118:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.171267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519257348830129:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2646
2025-11-11T17:21:42.161064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519260422606190:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16803855ns) {Configure:104(16798676ns) {ProcessConfigured:135(55794ns) {StartInitialHandshake:157(54302ns) {DropHandshakes:469(2300ns) {DropIncomingHandshake:441(356ns) DropOutgoingHandshake:456(342ns)} GenerateSessionVirtualId:988(354ns) PrepareNewSessionHandshake:476(2804ns) {SwitchToState:178(206ns)}}}}}
2025-11-11T17:21:42.154241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519255205468337:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(32835653ns) {Configure:104(32830697ns) {ProcessConfigured:135(59983ns) {StartInitialHandshake:157(57822ns) {DropHandshakes:469(3695ns) {DropIncomingHandshake:441(431ns) DropOutgoingHandshake:456(609ns)} GenerateSessionVirtualId:988(2410ns) PrepareNewSessionHandshake:476(1730ns) {SwitchToState:178(608ns)}}}}}
2025-11-11T17:21:42.175738Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519257348830118:2128] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:42.166738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435704:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.179976Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519257348830126:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.180330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519259141516287:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(51752314ns) {Configure:104(51746831ns) {ProcessConfigured:135(4032816ns) {StartInitialHandshake:157(4030089ns) {DropHandshakes:469(1470646ns) {DropIncomingHandshake:441(1059ns) DropOutgoingHandshake:456(1468ns)} GenerateSessionVirtualId:988(18097ns) PrepareNewSessionHandshake:476(2262ns) {SwitchToState:178(877ns)}}}}}
2025-11-11T17:21:42.166925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435705:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
2025-11-11T17:21:42.180038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519257348830112:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.167465Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519259500435704:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.184771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519259099234496:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(54623810ns) {Configure:104(54613192ns) {ProcessConfigured:135(5900370ns) {StartInitialHandshake:157(5897695ns) {DropHandshakes:469(2788ns) {DropIncomingHandshake:441(611ns) DropOutgoingHandshake:456(405ns)} GenerateSessionVirtualId:988(818ns) PrepareNewSessionHandshake:476(4340ns) {SwitchToState:178(2075ns)}}}}}
2025-11-11T17:21:42.191719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519256127638829:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21275774ns) {Configure:104(21272607ns) {ProcessConfigured:135(4923914ns) {StartInitialHandshake:157(4920978ns) {DropHandshakes:469(2589ns) {DropIncomingHandshake:441(525ns) DropOutgoingHandshake:456(529ns)} GenerateSessionVirtualId:988(537ns) PrepareNewSessionHandshake:476(3572ns) {SwitchToState:178(1420ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.167638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519255205468339:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.188134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519259099234503:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15206
2025-11-11T17:21:42.189735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491410:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.180075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:42.181013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435714:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
2025-11-11T17:21:42.200468Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519259140491410:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.182701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519257348830112:2107] [node 4] ICP22 created new session: [3:7571519257348830131:2107]
2025-11-11T17:21:42.188279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519259099234504:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15206
2025-11-11T17:21:42.207523Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519254846548986:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.188340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519259099234506:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15206
2025-11-11T17:21:42.189211Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234506:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.201661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519254804267137:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.206725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234503:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.208443Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234504:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.206514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519254845524035:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.182786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519257348830131:2107] [node 4] ICS09 handshake done sender: [3:7571519257348830118:2128] self: [3:7571519257348830117:0] peer: [4:7571519259500435713:0] socket: 103 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.206910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491411:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.183290Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519259500435704:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.222056Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234507:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.222092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519259099234507:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.207555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519254846548986:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.182833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519257348830131:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:42.222975Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519259099234507:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:42.183516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519255205468339:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.217183Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519259140491411:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.213650Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516286:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.182906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:42.223391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519259099234496:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.217341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519254845524036:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.183617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:42.213680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519259141516286:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.198358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519253053862736:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.183661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519255205468339:2050] [node 3] ICP22 created new session: [4:7571519259500435715:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.223444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:42.218064Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516289:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.218085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519259141516289:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.218588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519259141516297:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-11T17:21:42.218706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519259141516298:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-11T17:21:42.225850Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516298:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.228411Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519254846548986:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:42.228979Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519259141516289:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:42.223484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519259099234496:2108] [node 5] ICP22 created new session: [2:7571519259099234519:2108]
2025-11-11T17:21:42.221818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491415:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.228251Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:42.258235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519254845524036:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519259140491411:2117] held: no
2025-11-11T17:21:42.183721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519259500435715:2050] [node 3] ICS09 handshake done sender: [4:7571519259500435704:2113] self: [4:7571519259500435713:0] peer: [3:7571519257348830117:0] socket: 114 qp: -1
2025-11-11T17:21:42.183760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519259500435715:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:42.183790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:42.213322Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519259500435705:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.216686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435717:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
2025-11-11T17:21:42.228931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519259500435717:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.276117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519255205468338:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.253969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606197:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.202918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519257348830112:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(22291351ns) {HandleHandshakeStatus:329(22288783ns) {DropIncomingHandshake:441(327ns) DropOutgoingHandshake:456(869ns) SwitchToState:178(548ns) ProcessPendingSessionEvents:495(19226095ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19150363ns max# 7961396ns}}}
2025-11-11T17:21:42.223534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519259099234519:2108] [node 5] ICS09 handshake done sender: [2:7571519259099234507:2115] self: [2:7571519259099234505:0] peer: [5:7571519259140491413:0] socket: 128 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.299325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516297:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.203131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519257348830133:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2646
2025-11-11T17:21:42.254222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606198:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.299403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519259141516284:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.223569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519259099234519:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:42.254817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606201:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.295239Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519260422606197:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.295342Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519260422606201:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.295487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606205:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.295676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519256127638829:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.295749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519260422606190:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.300736Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519260422606205:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.300856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519256127638828:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.301021Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519260422606198:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.301112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519256127638890:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.323761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.299476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.207784Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:42.209285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519257348830112:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:42.223618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:42.326221Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519259500435707:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.299527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519259141516284:2108] [node 5] ICP22 created new session: [1:7571519259141516302:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.326045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491416:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.227295Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234500:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.217391Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519257348830133:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.326271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519259500435707:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.299576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519259141516302:2108] [node 5] ICS09 handshake done sender: [1:7571519259141516289:2112] self: [1:7571519259141516288:0] peer: [5:7571519259140491412:0] socket: 110 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.217514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519253053862737:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.230608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519257348830138:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2646
2025-11-11T17:21:42.244852Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519257348830133:2134] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:42.245060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519253053862737:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.245104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:42.245148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519253053862737:2049] [node 2] ICP22 created new session: [3:7571519257348830139:2049]
2025-11-11T17:21:42.245196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519257348830139:2049] [node 2] ICS09 handshake done sender: [3:7571519257348830133:2134] self: [3:7571519257348830136:0] peer: [2:7571519259099234499:0] socket: 132 qp: -1
2025-11-11T17:21:42.245227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519257348830139:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:42.245256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:42.278665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519257348830137:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2646
2025-11-11T17:21:42.296144Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519257348830126:2131] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:42.296698Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519257348830120:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.296719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519257348830120:2129] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.297146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519253053862736:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.297200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:42.297254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519253053862736:2048] [node 1] ICP22 created new session: [3:7571519257348830142:2048]
2025-11-11T17:21:42.297303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519257348830142:2048] [node 1] ICS09 handshake done sender: [3:7571519257348830126:2131] self: [3:7571519257348830135:0] peer: [1:7571519254846548982:0] socket: 112 qp: -1
2025-11-11T17:21:42.297337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519257348830142:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:42.297369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:42.322472Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519257348830120:2129] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:42.332869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519257348830114:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.227332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519259099234500:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.332918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:42.339712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606210:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.299608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519259141516302:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:42.332965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519257348830114:2117] [node 6] ICP22 created new session: [3:7571519257348830145:2117]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.368057Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519260422606197:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:42.366104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491414:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.368510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519256127638829:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.368608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:42.368665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519256127638829:2050] [node 3] ICP22 created new session: [6:7571519260422606213:2050]
2025-11-11T17:21:42.368716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519260422606213:2050] [node 3] ICS09 handshake done sender: [6:7571519260422606197:2115] self: [6:7571519260422606206:0] peer: [3:7571519257348830119:0] socket: 113 qp: -1
2025-11-11T17:21:42.368751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519260422606213:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:42.368782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.230324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519259099234523:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15206
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.299664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.370150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519254845524036:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(84559007ns) {HandleHandshakeStatus:395(84553018ns) {LogHandshakeFail:472(17367ns) {UpdateErrorStateLog:544(6304ns)} DropOutgoingHandshake:456(702ns) ProcessPendingSessionEvents:495(84444997ns) {unordered DropSessionEvent:512 num# 100 sum# 84357990ns max# 13137898ns}}}
2025-11-11T17:21:42.333013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519257348830145:2117] [node 6] ICS09 handshake done sender: [3:7571519257348830120:2129] self: [3:7571519257348830119:0] peer: [6:7571519260422606206:0] socket: 104 qp: -1
2025-11-11T17:21:42.366659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
2025-11-11T17:21:42.306122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519254846548923:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.310021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:42.317582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519254846548923:2049] [node 3] ICP22 created new session: [1:7571519259141516306:2049]
2025-11-11T17:21:42.317655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519259141516306:2049] [node 3] ICS09 handshake done sender: [1:7571519254846548986:2110] self: [1:7571519254846548982:0] peer: [3:7571519257348830135:0] socket: 108 qp: -1
2025-11-11T17:21:42.317701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519259141516306:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:42.317754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:42.331769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519254846548923:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(24826773ns) {HandleHandshakeStatus:329(24823483ns) {DropIncomingHandshake:441(544ns) DropOutgoingHandshake:456(784ns) SwitchToState:178(980ns) ProcessPendingSessionEvents:495(13020342ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12934287ns max# 5010748ns}}}
2025-11-11T17:21:42.332495Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519259141516286:2111] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:42.333496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519254846548923:2049] [node 3] ICP18 handshake (actor [1:7571519259141516298:2116]) is too old
2025-11-11T17:21:42.346686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519254846548922:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.358165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:42.360388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519254846548922:2048] [node 2] ICP22 created new session: [1:7571519259141516311:2048]
2025-11-11T17:21:42.373081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519259141516311:2048] [node 2] ICS09 handshake done sender: [1:7571519259141516286:2111] self: [1:7571519259141516285:0] peer: [2:7571519259099234515:0] socket: 109 qp: -1
2025-11-11T17:21:42.374301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519259141516311:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:42.377901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519259141516304:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-11T17:21:42.379476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519259141516305:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-11T17:21:42.385873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:42.371413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606212:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.238154Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519259099234500:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.333050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519257348830145:2117] [node 6] ICS10 traffic start
2025-11-11T17:21:42.371100Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519259140491415:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.241783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519254804267138:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.388875Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519259500435707:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:42.371534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491417:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.385857Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519259140491402:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.385908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519259140491402:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.395114Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519259140491410:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:42.395396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491418:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.409099Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519259140491411:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:42.333084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:42.351133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519257348830114:2117] [node 6] ICP35 event processing took too much time PendingConnection:278(17511489ns) {HandleHandshakeStatus:329(17507802ns) {DropIncomingHandshake:441(733ns) DropOutgoingHandshake:456(703ns) SwitchToState:178(709ns) ProcessPendingSessionEvents:495(17168852ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17090870ns max# 4841388ns}}}
2025-11-11T17:21:42.389343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519259500435697:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.398369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519257348830147:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2646
2025-11-11T17:21:42.241853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:42.421753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519259140491396:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.421858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519254845524035:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.421901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:42.421941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519254845524035:2048] [node 1] ICP22 created new session: [5:7571519259140491422:2048]
2025-11-11T17:21:42.417726Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516304:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.421987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519259140491422:2048] [node 1] ICS09 handshake done sender: [5:7571519259140491410:2116] self: [5:7571519259140491412:0] peer: [1:7571519259141516288:0] socket: 118 qp: -1
2025-11-11T17:21:42.422024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519259140491422:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:42.422056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:42.423745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519259140491418:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.423782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519254845524036:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.423816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:42.423855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519254845524036:2049] [node 2] ICP22 created new session: [5:7571519259140491424:2049]
2025-11-11T17:21:42.423904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519259140491424:2049] [node 2] ICS09 handshake done sender: [5:7571519259140491411:2117] self: [5:7571519259140491413:0] peer: [2:7571519259099234505:0] socket: 131 qp: -1
2025-11-11T17:21:42.423939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519259140491424:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:42.423963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:42.424656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519259140491417:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.241906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519254804267138:2049] [node 3] ICP22 created new session: [2:7571519259099234524:2049]
2025-11-11T17:21:42.389392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.418912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519254846548922:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(73500955ns) {HandleHandshakeStatus:329(71451872ns) {DropIncomingHandshake:441(742ns) DropOutgoingHandshake:456(1323ns) SwitchToState:178(1022ns) ProcessPendingSessionEvents:495(9688559ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9602169ns max# 4951005ns}}}
2025-11-11T17:21:42.389437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519259500435697:2109] [node 6] ICP22 created new session: [4:7571519259500435724:2109]
2025-11-11T17:21:42.389485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519259500435724:2109] [node 6] ICS09 handshake done sender: [4:7571519259500435707:2115] self: [4:7571519259500435706:0] peer: [6:7571519260422606207:0] socket: 122 qp: -1
2025-11-11T17:21:42.389526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519259500435724:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:42.389565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:42.403121Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519259500435717:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:42.404750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519255205468338:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.404805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:42.430287Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519259140491402:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:42.420711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519254846548922:2048] [node 2] ICP18 handshake (actor [1:7571519259141516304:2117]) is too old
2025-11-11T17:21:42.241953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519259099234524:2049] [node 3] ICS09 handshake done sender: [2:7571519259099234500:2110] self: [2:7571519259099234499:0] peer: [3:7571519257348830136:0] socket: 127 qp: -1
2025-11-11T17:21:42.242000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519259099234524:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:42.404854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519255205468338:2049] [node 2] ICP22 created new session: [4:7571519259500435726:2049]
2025-11-11T17:21:42.443733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519259140491395:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.430703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516305:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.242055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:42.404902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519259500435726:2049] [node 2] ICS09 handshake done sender: [4:7571519259500435717:2118] self: [4:7571519259500435720:0] peer: [2:7571519259099234508:0] socket: 130 qp: -1
2025-11-11T17:21:42.443808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:42.404939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519259500435726:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:42.404974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:42.431628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519255205468339:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(174535897ns) {HandleHandshakeStatus:329(174533463ns) {DropIncomingHandshake:441(817ns) DropOutgoingHandshake:456(4727ns) SwitchToState:178(50165ns) ProcessPendingSessionEvents:495(166425788ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 160208758ns max# 11833711ns}}}
2025-11-11T17:21:42.462548Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516291:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.462596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519259141516291:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.464495Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519259141516291:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:42.464838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519259141516283:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.464937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:42.464981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519259141516283:2107] [node 4] ICP22 created new session: [1:7571519259141516315:2107]
2025-11-11T17:21:42.465032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519259141516315:2107] [node 4] ICS09 handshake done sender: [1:7571519259141516291:2113] self: [1:7571519259141516290:0] peer: [4:7571519259500435732:0] socket: 134 qp: -1
2025-11-11T17:21:42.465065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519259141516315:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:42.465100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:42.301141Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234509:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.301172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519259099234509:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.317385Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234502:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.317441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519259099234502:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.318756Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519259099234523:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.323994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519259099234509:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:42.371513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519259099234497:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.371598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:42.372362Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519259099234502:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:42.372752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519259099234497:2107] [node 4] ICP22 created new session: [2:7571519259099234531:2107]
2025-11-11T17:21:42.395308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519259099234531:2107] [node 4] ICS09 handshake done sender: [2:7571519259099234509:2116] self: [2:7571519259099234508:0] peer: [4:7571519259500435720:0] socket: 129 qp: -1
2025-11-11T17:21:42.409174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519259099234531:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:42.410547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519259099234498:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.410647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:42.410705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519259099234498:2109] [node 6] ICP22 created new session: [2:7571519259099234532:2109]
2025-11-11T17:21:42.451905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519254845524037:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.466175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519256127638829:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(93355810ns) {HandleHandshakeStatus:329(93353382ns) {DropIncomingHandshake:441(1237ns) DropOutgoingHandshake:456(5705ns) SwitchToState:178(763ns) ProcessPendingSessionEvents:495(92833846ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 92689912ns max# 11629708ns}}}
2025-11-11T17:21:42.431769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435723:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
2025-11-11T17:21:42.410754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519259099234532:2109] [node 6] ICS09 handshake done sender: [2:7571519259099234502:2111] self: [2:7571519259099234501:0] peer: [6:7571519260422606208:0] socket: 107 qp: -1
2025-11-11T17:21:42.414698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:42.455881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519259140491395:2109] [node 6] ICP22 created new session: [5:7571519259140491426:2109]
2025-11-11T17:21:42.466828Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:42.432380Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519259500435709:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.417967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519259099234532:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:42.420491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:42.427525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519259099234498:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(16215734ns) {HandleHandshakeStatus:329(16211892ns) {DropIncomingHandshake:441(20526ns) DropOutgoingHandshake:456(580ns) SwitchToState:178(752ns) ProcessPendingSessionEvents:495(544962ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 490467ns max# 48316ns}}}
2025-11-11T17:21:42.427652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519259099234527:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15206
2025-11-11T17:21:42.429611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519259099234497:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(57354023ns) {HandleHandshakeStatus:329(57350094ns) {DropIncomingHandshake:441(1009ns) DropOutgoingHandshake:456(575ns) SwitchToState:178(493ns) ProcessPendingSessionEvents:495(2792276ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 2713233ns max# 2312869ns}}}
2025-11-11T17:21:42.437635Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:42.449991Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519259099234506:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:42.460405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519254804267137:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519259099234506:2114] held: no
2025-11-11T17:21:42.475333Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519257348830128:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.475399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519257348830128:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.432404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519259500435709:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.435337Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519259500435709:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:42.453996Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519259500435723:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.454052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519259500435696:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.454111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:42.454157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519259500435696:2108] [node 5] ICP22 created new session: [4:7571519259500435730:2108]
2025-11-11T17:21:42.454226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519259500435730:2108] [node 5] ICS09 handshake done sender: [4:7571519259500435709:2116] self: [4:7571519259500435708:0] peer: [5:7571519259140491421:0] socket: 138 qp: -1
2025-11-11T17:21:42.454267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519259500435730:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:42.454302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:42.461128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519255205468337:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.469130Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:42.470512Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519260422606205:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:42.462002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519259140491426:2109] [node 6] ICS09 handshake done sender: [5:7571519259140491402:2113] self: [5:7571519259140491401:0] peer: [6:7571519260422606209:0] socket: 106 qp: -1
2025-11-11T17:21:42.490790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519255205468337:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519259500435723:2120] held: no
2025-11-11T17:21:42.498060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519260422606201:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:42.490707Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519257348830128:2132] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:42.498450Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519257348830147:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.462059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519259140491426:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:42.462132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:42.471190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519259140491395:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(22200420ns) {HandleHandshakeStatus:329(22197359ns) {DropIncomingHandshake:441(684ns) DropOutgoingHandshake:456(1222ns) SwitchToState:178(1175ns) ProcessPendingSessionEvents:495(3649717ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3573297ns max# 3131905ns}}}
2025-11-11T17:21:42.473164Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:42.481164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519259140491396:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519259140491415:2119] held: no
2025-11-11T17:21:42.490819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571519259140491395:2109] [node 6] ICP18 handshake (actor [5:7571519259140491417:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.501475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519256127638828:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519260422606205:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.500332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519255205468337:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20055627ns) {HandleHandshakeStatus:395(18703940ns) {LogHandshakeFail:472(17195ns) {UpdateErrorStateLog:544(5170ns)} DropOutgoingHandshake:456(3193ns) ProcessPendingSessionEvents:495(8796859ns) {unordered DropSessionEvent:512 num# 100 sum# 540901ns max# 20638ns}}}
2025-11-11T17:21:42.499768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519254804267137:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(38566993ns) {HandleHandshakeStatus:395(38559254ns) {LogHandshakeFail:472(31044ns) {UpdateErrorStateLog:544(19189ns)} DropOutgoingHandshake:456(985ns) ProcessPendingSessionEvents:495(38463539ns) {unordered DropSessionEvent:512 num# 100 sum# 3892001ns max# 2123128ns}}}
2025-11-11T17:21:42.504085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519257348830113:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.500479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435733:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
2025-11-11T17:21:42.504912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:42.499972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519254804267137:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.502057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519256127638828:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.507101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491427:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.504965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519257348830113:2108] [node 5] ICP22 created new session: [3:7571519257348830150:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.502142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:42.514019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519259140491396:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(32120789ns) {HandleHandshakeStatus:395(32118126ns) {LogHandshakeFail:472(14899ns) {UpdateErrorStateLog:544(6187ns)} DropOutgoingHandshake:456(1434ns) ProcessPendingSessionEvents:495(30541395ns) {unordered DropSessionEvent:512 num# 100 sum# 30492087ns max# 11886929ns}}}
2025-11-11T17:21:42.513370Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519259500435723:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:42.500019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:42.518628Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:42.502185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519256127638828:2049] [node 2] ICP22 created new session: [6:7571519260422606218:2049]
2025-11-11T17:21:42.505020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519257348830150:2108] [node 5] ICS09 handshake done sender: [3:7571519257348830128:2132] self: [3:7571519257348830127:0] peer: [5:7571519259140491428:0] socket: 148 qp: -1
2025-11-11T17:21:42.500066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519254804267137:2048] [node 1] ICP22 created new session: [2:7571519259099234535:2048]
2025-11-11T17:21:42.505059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519257348830150:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:42.502227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519260422606218:2049] [node 2] ICS09 handshake done sender: [6:7571519260422606205:2119] self: [6:7571519260422606208:0] peer: [2:7571519259099234501:0] socket: 111 qp: -1
2025-11-11T17:21:42.500123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519259099234535:2048] [node 1] ICS09 handshake done sender: [2:7571519259099234506:2114] self: [2:7571519259099234515:0] peer: [1:7571519259141516285:0] socket: 119 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.516022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519255205468337:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.505090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:42.521643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519254845524037:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519259140491418:2122] held: no
2025-11-11T17:21:42.516074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.516131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519255205468337:2048] [node 1] ICP22 created new session: [4:7571519259500435735:2048]
2025-11-11T17:21:42.516184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519259500435735:2048] [node 1] ICS09 handshake done sender: [4:7571519259500435723:2120] self: [4:7571519259500435732:0] peer: [1:7571519259141516290:0] socket: 162 qp: -1
2025-11-11T17:21:42.516219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519259500435735:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:42.516249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:42.523240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519259500435734:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65028
2025-11-11T17:21:42.523307Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:42.523277Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519259140491415:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:42.502259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519260422606218:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:42.500162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519259099234535:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:42.502291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:42.500196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:42.523409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519259140491430:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53274
2025-11-11T17:21:42.503860Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:42.509230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519260422606190:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.509275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:42.510215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519260422606190:2110] [node 4] ICP22 created new session: [6:7571519260422606220:2110]
2025-11-11T17:21:42.510273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519260422606220:2110] [node 4] ICS09 handshake done sender: [6:7571519260422606201:2118] self: [6:7571519260422606207:0] peer: [4:7571519259500435706:0] socket: 135 qp: -1
2025-11-11T17:21:42.510308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519260422606220:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:42.510379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:42.525947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519259140491396:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.515357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606215:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.515526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606216:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.521258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519256127638890:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519260422606198:2116] held: no
2025-11-11T17:21:42.522549Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519260422606215:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.526506Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519259141516296:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:42.526543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519259141516296:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:42.526483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:42.522771Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519260422606198:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:42.524275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519256127638890:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.524345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:42.524377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519256127638890:2107] [node 5] ICP22 created new session: [6:7571519260422606222:2107]
2025-11-11T17:21:42.526520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519259140491396:2108] [node 4] ICP22 created new session: [5:7571519259140491431:2108]
2025-11-11T17:21:42.524409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519260422606222:2107] [node 5] ICS09 handshake done sender: [6:7571519260422606198:2116] self: [6:7571519260422606209:0] peer: [5:7571519259140491401:0] socket: 117 qp: -1
2025-11-11T17:21:42.524440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519260422606222:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:42.524466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:42.525921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519256127638827:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:42.526555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519259140491431:2108] [node 4] ICS09 handshake done sender: [5:7571519259140491415:2119] self: [5:7571519259140491421:0] peer: [4:7571519259500435708:0] socket: 139 qp: -1
2025-11-11T17:21:42.526586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519259140491431:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:42.526619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:42.526977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519259140491418:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:42.527305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519254845524037:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.527337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:42.527394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519254845524037:2050] [node 3] ICP22 created new session: [5:7571519259140491433:2050]
2025-11-11T17:21:42.527435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519259140491433:2050] [node 3] ICS09 handshake done sender: [5:7571519259140491418:2122] self: [5:7571519259140491428:0] peer: [3:7571519257348830127:0] socket: 150 qp: -1
2025-11-11T17:21:42.527462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519259140491433:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:42.527488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:42.534824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519260422606225:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37724
2025-11-11T17:21:42.541986Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519259141516296:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:42.542737Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519260422606215:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:42.542718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519259141516287:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:42.542816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:42.542866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519259141516287:2109] [node 6] ICP22 created new session: [1:7571519259141516318:2109]
2025-11-11T17:21:42.542911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519259141516318:2109] [node 6] ICS09 handshake done sender: [1:7571519259141516296:2114] self: [1:7571519259141516295:0] peer: [6:7571519260422606224:0] socket: 161 qp: -1
2025-11-11T17:21:42.543079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519256127638827:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:42.543129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:42.543182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519256127638827:2048] [node 1] ICP22 created new session: [6:7571519260422606226:2048]
2025-11-11T17:21:42.543225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519260422606226:2048] [node 1] ICS09 handshake done sender: [6:7571519260422606215:2123] self: [6:7571519260422606224:0] peer: [1:7571519259141516295:0] socket: 165 qp: -1
2025-11-11T17:21:42.543265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519260422606226:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:42.543340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:42.545487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519259141516318:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:42.545543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:42.585432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:42.585701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519257348830142:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-11-11T17:21:42.585740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519253053862736:2048] [node 1] ICP30 unregister session Session# [3:7571519257348830142:2048] VirtualId# [3:7571519257348830135:0]
2025-11-11T17:21:42.585778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519257348830142:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.585864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:42.585908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:42.589687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:42.589898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519259500435735:2048] [node 1] ICS01 socket: 162 reason# EndOfStream
2025-11-11T17:21:42.589939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519255205468337:2048] [node 1] ICP30 unregister session Session# [4:7571519259500435735:2048] VirtualId# [4:7571519259500435732:0]
2025-11-11T17:21:42.589972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519259500435735:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.590049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:42.590092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:42.589628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:42.589903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519259099234535:2048] [node 1] ICS01 socket: 119 reason# EndOfStream
2025-11-11T17:21:42.589938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519254804267137:2048] [node 1] ICP30 unregister session Session# [2:7571519259099234535:2048] VirtualId# [2:7571519259099234515:0]
2025-11-11T17:21:42.589972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519259099234535:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.590058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:42.590107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:42.591710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:42.591947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519260422606226:2048] [node 1] ICS07 socket disconnect 165 reason# EndOfStream
2025-11-11T17:21:42.591978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519260422606226:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.592071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:42.592121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519260422606226:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:42.594930Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address
2025-11-11T17:21:42.595133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519256127638827:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:42.599795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519260422606226:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:42.599845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519256127638827:2048] [node 1] ICP30 unregister session Session# [6:7571519260422606226:2048] VirtualId# [6:7571519260422606224:0]
2025-11-11T17:21:42.599898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:42.599935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519256127638827:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.599959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.593691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:42.593878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519259140491422:2048] [node 1] ICS07 socket disconnect 118 reason# EndOfStream
2025-11-11T17:21:42.593901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519259140491422:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.593972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:42.594016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519259140491422:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:42.600517Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address
2025-11-11T17:21:42.600829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519254845524035:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:42.600872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519259140491422:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:42.600901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519254845524035:2048] [node 1] ICP30 unregister session Session# [5:7571519259140491422:2048] VirtualId# [5:7571519259140491412:0]
2025-11-11T17:21:42.601010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:42.601045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519254845524035:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.601058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.611054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519256127638827:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:42.611170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519254845524035:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:42.669915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:42.670170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519257348830139:2049] [node 2] ICS01 socket: 132 reason# EndOfStream
2025-11-11T17:21:42.670216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519253053862737:2049] [node 2] ICP30 unregister session Session# [3:7571519257348830139:2049] VirtualId# [3:7571519257348830136:0]
2025-11-11T17:21:42.670253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519257348830139:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.670344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:42.670390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:42.675435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:42.675676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519260422606218:2049] [node 2] ICS01 socket: 111 reason# EndOfStream
2025-11-11T17:21:42.675722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519256127638828:2049] [node 2] ICP30 unregister session Session# [6:7571519260422606218:2049] VirtualId# [6:7571519260422606208:0]
2025-11-11T17:21:42.675757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519260422606218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.675842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:42.675889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:42.677862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:42.678158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519259500435726:2049] [node 2] ICS07 socket disconnect 130 reason# EndOfStream
2025-11-11T17:21:42.678209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519259500435726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.678536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:42.678353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:42.678810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519259140491424:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:21:42.678423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519259500435726:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:42.678874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519254845524036:2049] [node 2] ICP30 unregister session Session# [5:7571519259140491424:2049] VirtualId# [5:7571519259140491413:0]
2025-11-11T17:21:42.678928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519259140491424:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.679027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:42.679096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:42.689386Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Couldn't connect to any resolved address
2025-11-11T17:21:42.689724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519255205468338:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:42.689784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519259500435726:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:42.689820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519255205468338:2049] [node 2] ICP30 unregister session Session# [4:7571519259500435726:2049] VirtualId# [4:7571519259500435720:0]
2025-11-11T17:21:42.689924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:42.689953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519255205468338:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.689968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.699281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519255205468338:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:42.751343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:42.751603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519259140491433:2050] [node 3] ICS01 socket: 150 reason# EndOfStream
2025-11-11T17:21:42.751647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519254845524037:2050] [node 3] ICP30 unregister session Session# [5:7571519259140491433:2050] VirtualId# [5:7571519259140491428:0]
2025-11-11T17:21:42.751686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519259140491433:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.751780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:42.751824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:42.753748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:42.753748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:42.753992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519259500435715:2050] [node 3] ICS07 socket disconnect 114 reason# EndOfStream
2025-11-11T17:21:42.754029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519259500435715:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.754128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:42.754172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519259500435715:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:42.761860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519260422606213:2050] [node 3] ICS07 socket disconnect 113 reason# EndOfStream
2025-11-11T17:21:42.761912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519260422606213:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.762058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:42.765847Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address
2025-11-11T17:21:42.766187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519255205468339:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:42.766250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519259500435715:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:42.766288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519255205468339:2050] [node 3] ICP30 unregister session Session# [4:7571519259500435715:2050] VirtualId# [4:7571519259500435713:0]
2025-11-11T17:21:42.766401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:42.766433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519255205468339:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.766448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.762110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519260422606213:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:42.768381Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address
2025-11-11T17:21:42.772672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519256127638829:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:42.772755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519260422606213:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:42.772823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519256127638829:2050] [node 3] ICP30 unregister session Session# [6:7571519260422606213:2050] VirtualId# [6:7571519260422606206:0]
2025-11-11T17:21:42.772973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:42.773011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519256127638829:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.773028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.783520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519256127638829:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:42.806463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:42.806706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519259140491431:2108] [node 4] ICS01 socket: 139 reason# EndOfStream
2025-11-11T17:21:42.806758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519259140491396:2108] [node 4] ICP30 unregister session Session# [5:7571519259140491431:2108] VirtualId# [5:7571519259140491421:0]
2025-11-11T17:21:42.806798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519259140491431:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.806929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:42.806976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:42.809888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:42.810142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519260422606220:2110] [node 4] ICS07 socket disconnect 135 reason# EndOfStream
2025-11-11T17:21:42.810177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519260422606220:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.810310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:42.810367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519260422606220:2110] [node 4] ICS15 start handshake
2025-11-11T17:21:42.816627Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1278) Couldn't connect to any resolved address
2025-11-11T17:21:42.816988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519260422606190:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1278) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:42.817047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519260422606220:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:42.817084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519260422606190:2110] [node 4] ICP30 unregister session Session# [6:7571519260422606220:2110] VirtualId# [6:7571519260422606207:0]
2025-11-11T17:21:42.817185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:42.817219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519260422606190:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.817232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:42.828199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519260422606190:2110] [node 4] ICP33 wake up from error state
2025-11-11T17:21:42.877797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:42.878050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519260422606222:2107] [node 5] ICS01 socket: 117 reason# EndOfStream
2025-11-11T17:21:42.878090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519256127638890:2107] [node 5] ICP30 unregister session Session# [6:7571519260422606222:2107] VirtualId# [6:7571519260422606209:0]
2025-11-11T17:21:42.878124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519260422606222:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:42.878263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:42.878317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:42.982886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519260391221958:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:42.982945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519260391221959:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:43.003448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519265239784879:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:43.004054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519265239784880:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:43.025920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519262681819442:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:43.025918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519262681819443:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:43.049122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519263119004081:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:43.049164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519263119004082:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:43.049511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519263119004080:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:43.070600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519262338214005:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:43.070641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519262338214006:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:43.070681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519262338214007:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:43.091114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519264897926251:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:43.091168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519264897926253:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:43.091261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519264897926252:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:43.220057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519264897926299:2091] [node 4] ICP01 ready to work
2025-11-11T17:21:43.217650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519264686189316:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:43.273667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519264686189318:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:43.304917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519262681819508:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:43.314340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519263119004144:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:43.308719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519265239784943:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:43.299974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519264686189319:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:43.300186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519264897926303:2093] [node 5] ICP01 ready to work
2025-11-11T17:21:43.305241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519262681819507:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:43.314404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519265239784942:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:43.308547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519262681819506:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:43.326079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519265239784944:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:43.329655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519262338214069:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:43.344242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519260391221958:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(33048660ns) {Configure:104(33040727ns) {ProcessConfigured:135(52649ns) {StartInitialHandshake:157(51294ns) {DropHandshakes:469(1947ns) {DropIncomingHandshake:441(516ns) DropOutgoingHandshake:456(230ns)} GenerateSessionVirtualId:988(955ns) PrepareNewSessionHandshake:476(2095ns) {SwitchToState:178(487ns)}}}}}
2025-11-11T17:21:43.344584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519263119004145:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:43.344644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519262338214070:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:43.351678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519260391221959:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(132147843ns) {Configure:104(132144074ns) {ProcessConfigured:135(5902588ns) {StartInitialHandshake:157(5899250ns) {DropHandshakes:469(1597ns) {DropIncomingHandshake:441(326ns) DropOutgoingHandshake:456(206ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:476(1596098ns) {SwitchToState:178(2854ns)}}}}}
2025-11-11T17:21:43.353861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519263119004080:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(98475406ns) {Configure:104(98470626ns) {ProcessConfigured:135(2195838ns) {StartInitialHandshake:157(2194075ns) {DropHandshakes:469(3199ns) {DropIncomingHandshake:441(921ns) DropOutgoingHandshake:456(365ns)} GenerateSessionVirtualId:988(779ns) PrepareNewSessionHandshake:476(2543ns) {SwitchToState:178(1174ns)}}}}}
2025-11-11T17:21:43.364835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519264897926252:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(140435250ns) {Configure:104(140424134ns) {ProcessConfigured:135(5923903ns) {StartInitialHandshake:157(5922036ns) {DropHandshakes:469(4016ns) {DropIncomingHandshake:441(612ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(879ns) PrepareNewSessionHandshake:476(2776ns) {SwitchToState:178(521ns)}}}}}
2025-11-11T17:21:43.375479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519262681819442:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34374170ns) {Configure:104(34365960ns) {ProcessConfigured:135(4482373ns) {StartInitialHandshake:157(4480591ns) {DropHandshakes:469(6403ns) {DropIncomingHandshake:441(218ns) DropOutgoingHandshake:456(246ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:476(3396ns) {SwitchToState:178(2322ns)}}}}}
2025-11-11T17:21:43.386526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519264897926299:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23465810ns) {Configure:104(23458809ns) {ProcessConfigured:135(1048939ns) {StartInitialHandshake:157(1046295ns) {DropHandshakes:469(3210ns) {DropIncomingHandshake:441(924ns) DropOutgoingHandshake:456(421ns)} GenerateSessionVirtualId:988(4140ns) PrepareNewSessionHandshake:476(2856ns) {SwitchToState:178(1426ns)}}}}}
2025-11-11T17:21:43.386848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519265239784879:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27375580ns) {Configure:104(27370482ns) {ProcessConfigured:135(2802603ns) {StartInitialHandshake:157(2800304ns) {DropHandshakes:469(4182ns) {DropIncomingHandshake:441(744ns) DropOutgoingHandshake:456(482ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:476(2530ns) {SwitchToState:178(1009ns)}}}}}
2025-11-11T17:21:43.388081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519262338214006:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(33706984ns) {Configure:104(33703449ns) {ProcessConfigured:135(6762113ns) {StartInitialHandshake:157(6759998ns) {DropHandshakes:469(2907ns) {DropIncomingHandshake:441(576ns) DropOutgoingHandshake:456(417ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:476(2787ns) {SwitchToState:178(897ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.394024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519262681819508:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(33236596ns) {Configure:104(33232888ns) {ProcessConfigured:135(1197745ns) {StartInitialHandshake:157(1185049ns) {DropHandshakes:469(3040ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(275ns)} GenerateSessionVirtualId:988(828ns) PrepareNewSessionHandshake:476(1497ns) {SwitchToState:178(635ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.415711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926329:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.408607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519264686189319:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38500494ns) {Configure:104(37084929ns) {ProcessConfigured:135(56084ns) {StartInitialHandshake:157(53809ns) {DropHandshakes:469(1740ns) {DropIncomingHandshake:441(330ns) DropOutgoingHandshake:456(251ns)} GenerateSessionVirtualId:988(569ns) PrepareNewSessionHandshake:476(4518ns) {SwitchToState:178(747ns)}}}}}
2025-11-11T17:21:43.411480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519262681819518:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6478
2025-11-11T17:21:43.414995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519265239784943:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38410737ns) {Configure:104(38406166ns) {ProcessConfigured:135(9626249ns) {StartInitialHandshake:157(9623309ns) {DropHandshakes:469(656ns) {DropIncomingHandshake:441(125ns) DropOutgoingHandshake:456(203ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:476(3960ns) {SwitchToState:178(1630ns)}}}}}
2025-11-11T17:21:43.414744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519262338214007:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(39912094ns) {Configure:104(39907294ns) {ProcessConfigured:135(5940863ns) {StartInitialHandshake:157(5938607ns) {DropHandshakes:469(419046ns) {DropIncomingHandshake:441(815ns) DropOutgoingHandshake:456(238ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:476(2054ns) {SwitchToState:178(586ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.415094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519262681819443:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(41533019ns) {Configure:104(41528852ns) {ProcessConfigured:135(5744898ns) {StartInitialHandshake:157(5743038ns) {DropHandshakes:469(1980ns) {DropIncomingHandshake:441(830ns) DropOutgoingHandshake:456(284ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:476(2944ns) {SwitchToState:178(942ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.429707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519264897926253:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(56718750ns) {Configure:104(56713974ns) {ProcessConfigured:135(15936170ns) {StartInitialHandshake:157(15933494ns) {DropHandshakes:469(2859ns) {DropIncomingHandshake:441(779ns) DropOutgoingHandshake:456(406ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:476(3164ns) {SwitchToState:178(987ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.428984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519264686189318:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(49569805ns) {Configure:104(49564716ns) {ProcessConfigured:135(9536364ns) {StartInitialHandshake:157(9534127ns) {DropHandshakes:469(1454ns) {DropIncomingHandshake:441(519ns) DropOutgoingHandshake:456(290ns)} GenerateSessionVirtualId:988(416ns) PrepareNewSessionHandshake:476(2422ns) {SwitchToState:178(185ns)}}}}}
2025-11-11T17:21:43.421843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519262681819507:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(39261262ns) {Configure:104(39256525ns) {ProcessConfigured:135(8780030ns) {StartInitialHandshake:157(8777325ns) {DropHandshakes:469(14555ns) {DropIncomingHandshake:441(843ns) DropOutgoingHandshake:456(11908ns)} GenerateSessionVirtualId:988(881ns) PrepareNewSessionHandshake:476(2029ns) {SwitchToState:178(599ns)}}}}}
2025-11-11T17:21:43.446820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004160:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
2025-11-11T17:21:43.447571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004159:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.446417Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519264897926329:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.447079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519264897926253:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.448215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519265239784957:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41997
2025-11-11T17:21:43.448655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519265239784960:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41997
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.455680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214077:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.455864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214078:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.455981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214076:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.454951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519264686189324:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.462417Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519263119004159:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.456239Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519262681819512:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.454192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519265239784956:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41997
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.456288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519262681819512:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.455093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519264686189329:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60245
2025-11-11T17:21:43.478033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926331:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.468743Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519263119004160:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.478498Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519262338214076:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.454321Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784960:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.481264Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519262338214077:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.469044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519264686189316:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(62179406ns) {Configure:104(62172009ns) {ProcessConfigured:135(998914ns) {StartInitialHandshake:157(996023ns) {DropHandshakes:469(5646ns) {DropIncomingHandshake:441(1095ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(925ns) PrepareNewSessionHandshake:476(2889ns) {SwitchToState:178(859ns)}}}}}
2025-11-11T17:21:43.466546Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519262681819518:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.466898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519262681819443:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.467347Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519262681819512:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:43.467698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519262681819508:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.467766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:43.467816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519262681819508:2110] [node 6] ICP22 created new session: [3:7571519262681819525:2110]
2025-11-11T17:21:43.467863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519262681819525:2110] [node 6] ICS09 handshake done sender: [3:7571519262681819512:2112] self: [3:7571519262681819511:0] peer: [6:7571519264897926330:0] socket: 127 qp: -1
2025-11-11T17:21:43.467902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519262681819525:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:43.467938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:43.514305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519262681819527:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6478
2025-11-11T17:21:43.490289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519263119004081:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.478917Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784948:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.522216Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519262681819527:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.509915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519263119004151:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.488772Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519262338214078:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.490400Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189324:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.503315Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189329:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.522295Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519264897926331:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.478958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519265239784948:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.532013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519262681819442:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.492031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519262338214005:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(44242534ns) {Configure:104(44236155ns) {ProcessConfigured:135(57287ns) {StartInitialHandshake:157(54865ns) {DropHandshakes:469(8415ns) {DropIncomingHandshake:441(827ns) DropOutgoingHandshake:456(683ns)} GenerateSessionVirtualId:988(1159ns) PrepareNewSessionHandshake:476(1262ns) {SwitchToState:178(274ns)}}}}}
2025-11-11T17:21:43.526469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519264897926251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.509955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519263119004151:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.484952Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784957:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.529646Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189322:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.537468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926332:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.514964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
2025-11-11T17:21:43.529678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519264686189322:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.492299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519262338214069:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.485184Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784956:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.541449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926333:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.531588Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519264686189322:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:43.492395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519262338214006:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.533873Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519263119004151:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:43.485582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519265239784962:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41997
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.545566Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519264897926333:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.545985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926335:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.546069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519264897926303:2093] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.546125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926336:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.539290Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519263119004164:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.500802Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519265239784948:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:43.544120Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:43.533473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189327:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.550235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214091:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.539329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519263119004145:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.550190Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519264897926329:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:43.545007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519262681819528:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6478
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.533503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519264686189327:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.509865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519265239784880:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(19264490ns) {IncomingHandshake:261(19262459ns) {DropIncomingHandshake:441(658ns) PrepareNewSessionHandshake:476(4153906ns) {SwitchToState:178(7993ns)} GenerateSessionVirtualId:988(1841ns) IssueIncomingHandshakeReply:193(9630ns)}}
2025-11-11T17:21:43.552245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214090:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.554130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519264897926253:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.550371Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519262681819517:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.534104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519260391221958:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.539668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004165:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
2025-11-11T17:21:43.552928Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519262338214080:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.526928Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784946:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.550407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519262681819517:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.554184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.541847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004163:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
2025-11-11T17:21:43.526958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519265239784946:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.534152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:43.552961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519262338214080:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.562905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519264897926253:2050] [node 3] ICP22 created new session: [6:7571519264897926338:2050]
2025-11-11T17:21:43.542078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519263119004082:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.558262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519262681819443:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519262681819518:2115] held: no
2025-11-11T17:21:43.570488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926340:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.534200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519260391221958:2048] [node 2] ICP22 created new session: [1:7571519264686189338:2048]
2025-11-11T17:21:43.528951Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784962:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.529079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519265239784966:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41997
2025-11-11T17:21:43.529349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519265239784879:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.530551Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784950:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.530580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519265239784950:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.532484Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519265239784946:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:43.536622Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784966:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.536668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519265239784880:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.536720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:43.536763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519265239784880:2049] [node 3] ICP22 created new session: [2:7571519265239784969:2049]
2025-11-11T17:21:43.536811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519265239784969:2049] [node 3] ICS09 handshake done sender: [2:7571519265239784948:2111] self: [2:7571519265239784947:0] peer: [3:7571519262681819524:0] socket: 118 qp: -1
2025-11-11T17:21:43.537803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519265239784942:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.537881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:43.537923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519265239784942:2107] [node 4] ICP22 created new session: [2:7571519265239784971:2107]
2025-11-11T17:21:43.537971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519265239784971:2107] [node 4] ICS09 handshake done sender: [2:7571519265239784946:2110] self: [2:7571519265239784945:0] peer: [4:7571519263119004162:0] socket: 119 qp: -1
2025-11-11T17:21:43.538003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519265239784971:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:43.538035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:43.538435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519265239784969:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:43.539028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:43.563341Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519265239784950:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:43.564222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519265239784944:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.564273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:43.564324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519265239784944:2108] [node 5] ICP22 created new session: [2:7571519265239784978:2108]
2025-11-11T17:21:43.564367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519265239784978:2108] [node 5] ICS09 handshake done sender: [2:7571519265239784950:2112] self: [2:7571519265239784949:0] peer: [5:7571519262338214089:0] socket: 120 qp: -1
2025-11-11T17:21:43.564402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519265239784978:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:43.564441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:43.593479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519265239784970:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41997
2025-11-11T17:21:43.614331Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519265239784962:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:43.614909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519265239784959:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.614932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519265239784959:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.623187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519265239784879:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.623251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:43.570557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926341:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.542706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:43.593547Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519262338214091:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.559270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519262681819518:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:43.534256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519264686189338:2048] [node 2] ICS09 handshake done sender: [1:7571519264686189322:2110] self: [1:7571519264686189321:0] peer: [2:7571519265239784968:0] socket: 133 qp: -1
2025-11-11T17:21:43.571013Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519264897926331:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:43.534289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519264686189338:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:43.594184Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.632431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519265239784879:2048] [node 1] ICP22 created new session: [2:7571519265239784980:2048]
2025-11-11T17:21:43.544675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519263119004145:2108] [node 5] ICP22 created new session: [4:7571519263119004167:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.567636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519262681819443:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.572548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519264897926251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.579781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:43.582869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519264897926251:2048] [node 1] ICP22 created new session: [6:7571519264897926342:2048]
2025-11-11T17:21:43.592733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519264897926338:2050] [node 3] ICS09 handshake done sender: [6:7571519264897926329:2115] self: [6:7571519264897926330:0] peer: [3:7571519262681819511:0] socket: 129 qp: -1
2025-11-11T17:21:43.592796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519264897926338:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:43.592843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:43.594373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519264897926342:2048] [node 1] ICS09 handshake done sender: [6:7571519264897926331:2116] self: [6:7571519264897926334:0] peer: [1:7571519264686189326:0] socket: 137 qp: -1
2025-11-11T17:21:43.594410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519264897926342:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:43.595890Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519264897926333:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:43.596186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519264897926303:2093] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.596233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:43.596283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519264897926303:2093] [node 5] ICP22 created new session: [6:7571519264897926344:2093]
2025-11-11T17:21:43.596330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519264897926344:2093] [node 5] ICS09 handshake done sender: [6:7571519264897926333:2118] self: [6:7571519264897926337:0] peer: [5:7571519262338214079:0] socket: 143 qp: -1
2025-11-11T17:21:43.596375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519264897926344:2093] [node 5] ICS10 traffic start
2025-11-11T17:21:43.596425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:43.605716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:43.614542Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519264897926341:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.614660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519264897926252:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.658856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519264897926253:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(103874766ns) {HandleHandshakeStatus:329(103871373ns) {DropIncomingHandshake:441(805ns) DropOutgoingHandshake:456(6023ns) SwitchToState:178(1072ns) ProcessPendingSessionEvents:495(40164245ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40078268ns max# 24135411ns}}}
2025-11-11T17:21:43.660412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926348:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.661523Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519264897926341:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:43.661822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519264897926252:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.661875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:43.534322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.655416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519265239784980:2048] [node 1] ICS09 handshake done sender: [2:7571519265239784962:2118] self: [2:7571519265239784968:0] peer: [1:7571519264686189321:0] socket: 138 qp: -1
2025-11-11T17:21:43.594700Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519262338214080:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:43.567699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:43.546695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519263119004167:2108] [node 5] ICS09 handshake done sender: [4:7571519263119004151:2113] self: [4:7571519263119004150:0] peer: [5:7571519262338214087:0] socket: 109 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.539023Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189323:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.688153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519264897926252:2049] [node 2] ICP22 created new session: [6:7571519264897926349:2049]
2025-11-11T17:21:43.550939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519263119004167:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:43.594967Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519262338214076:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:43.657216Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519265239784959:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:43.550986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:43.688668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519264897926349:2049] [node 2] ICS09 handshake done sender: [6:7571519264897926341:2122] self: [6:7571519264897926347:0] peer: [2:7571519265239784958:0] socket: 169 qp: -1
2025-11-11T17:21:43.657327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519265239784980:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:43.567741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519262681819443:2049] [node 2] ICP22 created new session: [3:7571519262681819534:2049]
2025-11-11T17:21:43.539051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519264686189323:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.552765Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519263119004163:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.622754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519262338214006:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519262338214078:2115] held: no
2025-11-11T17:21:43.627002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519262338214069:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.627070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:43.627113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519262338214069:2108] [node 4] ICP22 created new session: [5:7571519262338214095:2108]
2025-11-11T17:21:43.627164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519262338214095:2108] [node 4] ICS09 handshake done sender: [5:7571519262338214076:2113] self: [5:7571519262338214087:0] peer: [4:7571519263119004150:0] socket: 121 qp: -1
2025-11-11T17:21:43.627201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519262338214095:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:43.627260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:43.628343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519262338214070:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.628394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:43.628429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519262338214070:2109] [node 6] ICP22 created new session: [5:7571519262338214097:2109]
2025-11-11T17:21:43.628468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519262338214097:2109] [node 6] ICS09 handshake done sender: [5:7571519262338214080:2116] self: [5:7571519262338214079:0] peer: [6:7571519264897926337:0] socket: 140 qp: -1
2025-11-11T17:21:43.628498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519262338214097:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:43.628529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:43.651129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519262338214006:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(27596384ns) {HandleHandshakeStatus:395(27593503ns) {LogHandshakeFail:472(12974ns) {UpdateErrorStateLog:544(3492ns)} DropOutgoingHandshake:456(840ns) ProcessPendingSessionEvents:495(27517937ns) {unordered DropSessionEvent:512 num# 100 sum# 27476053ns max# 12652429ns}}}
2025-11-11T17:21:43.656692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519262338214005:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.675928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214093:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.689861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214094:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.712334Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519262338214094:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.712448Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519262338214078:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:43.567782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519262681819534:2049] [node 2] ICS09 handshake done sender: [3:7571519262681819518:2115] self: [3:7571519262681819524:0] peer: [2:7571519265239784947:0] socket: 125 qp: -1
2025-11-11T17:21:43.688709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519264897926349:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:43.657370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:43.545084Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519264686189323:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:43.552464Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519264686189327:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:43.559563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519264686189319:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.559630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:43.559679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519264686189319:2109] [node 6] ICP22 created new session: [1:7571519264686189343:2109]
2025-11-11T17:21:43.559732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519264686189343:2109] [node 6] ICS09 handshake done sender: [1:7571519264686189327:2113] self: [1:7571519264686189326:0] peer: [6:7571519264897926334:0] socket: 134 qp: -1
2025-11-11T17:21:43.559770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519264686189343:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:43.559811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:43.563033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519260391221959:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.563094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:43.567328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519260391221959:2049] [node 3] ICP22 created new session: [1:7571519264686189346:2049]
2025-11-11T17:21:43.582694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519264686189346:2049] [node 3] ICS09 handshake done sender: [1:7571519264686189323:2111] self: [1:7571519264686189320:0] peer: [3:7571519262681819530:0] socket: 132 qp: -1
2025-11-11T17:21:43.589051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519264686189346:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:43.589116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:43.591576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519264686189347:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60245
2025-11-11T17:21:43.593890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519260391221959:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(32130978ns) {HandleHandshakeStatus:329(32127914ns) {DropIncomingHandshake:441(14147ns) DropOutgoingHandshake:456(263ns) SwitchToState:178(262ns) ProcessPendingSessionEvents:495(527510ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 445320ns max# 57615ns}}}
2025-11-11T17:21:43.594632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189347:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.617691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519264686189348:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60245
2025-11-11T17:21:43.636449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519260391221958:2048] [node 2] ICP18 handshake (actor [1:7571519264686189347:2117]) is too old
2025-11-11T17:21:43.636536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [1:7571519260391221958:2048] [node 2] ICP35 event processing took too much time StateWork:293(22761316ns) {IncomingHandshake:261(19567328ns)}
2025-11-11T17:21:43.655116Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:43.701822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519264686189319:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:43.711313Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189328:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.711340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519264686189328:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.553043Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519263119004160:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:43.719325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519262338214006:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.728361Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519264686189328:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:43.688753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.711255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519265239784880:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(63355945ns) {HandleHandshakeStatus:329(63353922ns) {DropIncomingHandshake:441(13057ns) DropOutgoingHandshake:456(636ns) SwitchToState:178(667ns) ProcessPendingSessionEvents:495(60185966ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60081392ns max# 10567897ns}}}
2025-11-11T17:21:43.567819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519262681819534:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:43.581077Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:43.594351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519263119004081:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.600068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:43.625657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519263119004081:2049] [node 2] ICP22 created new session: [4:7571519263119004171:2049]
2025-11-11T17:21:43.645662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519263119004171:2049] [node 2] ICS09 handshake done sender: [4:7571519263119004160:2117] self: [4:7571519263119004162:0] peer: [2:7571519265239784945:0] socket: 122 qp: -1
2025-11-11T17:21:43.645716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519263119004171:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:43.645766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:43.695036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519263119004081:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(99750705ns) {HandleHandshakeStatus:329(99747389ns) {DropIncomingHandshake:441(1518ns) DropOutgoingHandshake:456(9621ns) SwitchToState:178(1507ns) ProcessPendingSessionEvents:495(36867006ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36793681ns max# 30013886ns}}}
2025-11-11T17:21:43.695617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519263119004081:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:43.696765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004173:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
2025-11-11T17:21:43.727086Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519263119004156:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.727116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519263119004156:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.761713Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519263119004164:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:43.765645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004172:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
2025-11-11T17:21:43.719412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:43.567857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:43.749981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519264686189351:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60245
2025-11-11T17:21:43.698503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519264897926252:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(27512221ns) {HandleHandshakeStatus:329(27510198ns) {DropIncomingHandshake:441(946ns) DropOutgoingHandshake:456(5544ns) SwitchToState:178(606ns) ProcessPendingSessionEvents:495(471690ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 401856ns max# 40389ns}}}
2025-11-11T17:21:43.719974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519262338214006:2049] [node 2] ICP22 created new session: [5:7571519262338214102:2049]
2025-11-11T17:21:43.711316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519265239784943:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.764383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519264686189318:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.568599Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519262681819517:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:43.700000Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519264897926335:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.790114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519263119004172:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.711379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.593713Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:43.742076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519262338214102:2049] [node 2] ICS09 handshake done sender: [5:7571519262338214078:2115] self: [5:7571519262338214089:0] peer: [2:7571519265239784949:0] socket: 124 qp: -1
2025-11-11T17:21:43.700562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519264897926299:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.793756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:43.790181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519263119004082:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.754302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519262338214007:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.711429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519265239784943:2109] [node 6] ICP22 created new session: [2:7571519265239784983:2109]
2025-11-11T17:21:43.790246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:43.790302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519263119004082:2050] [node 3] ICP22 created new session: [4:7571519263119004177:2050]
2025-11-11T17:21:43.790351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519263119004177:2050] [node 3] ICS09 handshake done sender: [4:7571519263119004164:2119] self: [4:7571519263119004169:0] peer: [3:7571519262681819516:0] socket: 149 qp: -1
2025-11-11T17:21:43.790391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519263119004177:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:43.790432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:43.798250Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519263119004156:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:43.814312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519263119004080:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:43.821951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519263119004144:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.822031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:43.822083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519263119004144:2110] [node 6] ICP22 created new session: [4:7571519263119004180:2110]
2025-11-11T17:21:43.822143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519263119004180:2110] [node 6] ICS09 handshake done sender: [4:7571519263119004156:2114] self: [4:7571519263119004155:0] peer: [6:7571519264897926351:0] socket: 130 qp: -1
2025-11-11T17:21:43.822191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519263119004180:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:43.822228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:43.597772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519262681819506:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.793820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519264686189318:2108] [node 5] ICP22 created new session: [1:7571519264686189352:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.719643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519265239784983:2109] [node 6] ICS09 handshake done sender: [2:7571519265239784959:2116] self: [2:7571519265239784958:0] peer: [6:7571519264897926347:0] socket: 165 qp: -1
2025-11-11T17:21:43.597811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:43.754815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214103:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.793899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519264686189352:2108] [node 5] ICS09 handshake done sender: [1:7571519264686189328:2114] self: [1:7571519264686189325:0] peer: [5:7571519262338214099:0] socket: 155 qp: -1
2025-11-11T17:21:43.851476Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:43.719704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519265239784983:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:43.727987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:43.793632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519265239784943:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(66877572ns) {HandleHandshakeStatus:329(66874603ns) {DropIncomingHandshake:441(13138ns) DropOutgoingHandshake:456(736ns) SwitchToState:178(537ns) ProcessPendingSessionEvents:495(15923854ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15852707ns max# 15415608ns}}}
2025-11-11T17:21:43.755435Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519262338214091:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:43.597857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519262681819506:2107] [node 4] ICP22 created new session: [3:7571519262681819536:2107]
2025-11-11T17:21:43.597898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519262681819536:2107] [node 4] ICS09 handshake done sender: [3:7571519262681819517:2114] self: [3:7571519262681819516:0] peer: [4:7571519263119004169:0] socket: 148 qp: -1
2025-11-11T17:21:43.793934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519264686189352:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:43.793988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:43.795500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519264686189318:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(30298720ns) {HandleHandshakeStatus:329(30295987ns) {DropIncomingHandshake:441(489ns) DropOutgoingHandshake:456(653ns) SwitchToState:178(4354ns) ProcessPendingSessionEvents:495(565209ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 485473ns max# 57368ns}}}
2025-11-11T17:21:43.857137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519264897926352:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60146
2025-11-11T17:21:43.813768Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 5] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:43.823568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7571519264686189318:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:43.867568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519264897926299:2091] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519264897926335:2119] held: no
2025-11-11T17:21:43.845688Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519264686189331:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.845723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519264686189331:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.854361Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519264686189331:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:43.854741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519264686189316:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.854839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:43.854887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519264686189316:2107] [node 4] ICP22 created new session: [1:7571519264686189355:2107]
2025-11-11T17:21:43.854958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519264686189355:2107] [node 4] ICS09 handshake done sender: [1:7571519264686189331:2116] self: [1:7571519264686189330:0] peer: [4:7571519263119004179:0] socket: 162 qp: -1
2025-11-11T17:21:43.855005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519264686189355:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:43.855043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:43.761959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519262338214005:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.762044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:43.767043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519262338214102:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:43.781738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519262338214005:2048] [node 1] ICP22 created new session: [5:7571519262338214105:2048]
2025-11-11T17:21:43.792366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:43.792559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519262338214105:2048] [node 1] ICS09 handshake done sender: [5:7571519262338214091:2120] self: [5:7571519262338214099:0] peer: [1:7571519264686189325:0] socket: 156 qp: -1
2025-11-11T17:21:43.799472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519262338214105:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:43.803893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:43.807606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519262338214108:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-11-11T17:21:43.818069Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519262338214094:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:43.823412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519262338214006:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(97234241ns) {HandleHandshakeStatus:329(90733596ns) {DropIncomingHandshake:441(1009ns) DropOutgoingHandshake:456(749ns) SwitchToState:178(569ns) ProcessPendingSessionEvents:495(2331ns)}}
2025-11-11T17:21:43.841834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519262338214007:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.852316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:43.852369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519262338214007:2050] [node 3] ICP22 created new session: [5:7571519262338214109:2050]
2025-11-11T17:21:43.857527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519262338214109:2050] [node 3] ICS09 handshake done sender: [5:7571519262338214094:2122] self: [5:7571519262338214104:0] peer: [3:7571519262681819515:0] socket: 172 qp: -1
2025-11-11T17:21:43.867104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519262338214109:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:43.864442Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:43.868447Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519264897926335:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:43.597930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519262681819536:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:43.869024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:43.869306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519264897926299:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.597959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:43.866126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519263119004080:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519263119004172:2121] held: no
2025-11-11T17:21:43.598893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519262681819442:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519262681819527:2117] held: no
2025-11-11T17:21:43.616820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519262681819531:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6478
2025-11-11T17:21:43.870023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:43.873309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519265239784879:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(250766507ns) {HandleHandshakeStatus:329(249372352ns) {DropIncomingHandshake:441(1541ns) DropOutgoingHandshake:456(7607ns) SwitchToState:178(751ns) ProcessPendingSessionEvents:495(184108773ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 183964060ns max# 17696995ns}}}
2025-11-11T17:21:43.868794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519263119004182:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52824
2025-11-11T17:21:43.872062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519263119004145:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(318181895ns) {HandleHandshakeStatus:329(318180074ns) {DropIncomingHandshake:441(262ns) DropOutgoingHandshake:456(2619ns) SwitchToState:178(268ns) ProcessPendingSessionEvents:495(277581113ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 263084718ns max# 19439630ns}}}
2025-11-11T17:21:43.872419Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519263119004172:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:43.873085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519263119004145:2108] [node 5] ICP18 handshake (actor [4:7571519263119004163:2118]) is too old
2025-11-11T17:21:43.870085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519264897926299:2091] [node 4] ICP22 created new session: [6:7571519264897926353:2091]
2025-11-11T17:21:43.646450Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519262681819527:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:43.876539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519262338214005:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(117762283ns) {HandleHandshakeStatus:329(117759424ns) {DropIncomingHandshake:441(2151ns) DropOutgoingHandshake:456(11591ns) SwitchToState:178(994ns) ProcessPendingSessionEvents:495(60865663ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60774739ns max# 19003323ns}}}
2025-11-11T17:21:43.661873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519262681819442:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.879760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519263119004080:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:43.870130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519264897926353:2091] [node 4] ICS09 handshake done sender: [6:7571519264897926335:2119] self: [6:7571519264897926351:0] peer: [4:7571519263119004155:0] socket: 159 qp: -1
2025-11-11T17:21:43.870403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519264897926353:2091] [node 4] ICS10 traffic start
2025-11-11T17:21:43.870797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:43.877919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519264897926251:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(295941798ns) {HandleHandshakeStatus:329(295939776ns) {DropIncomingHandshake:441(647ns) DropOutgoingHandshake:456(4749ns) SwitchToState:178(518ns) ProcessPendingSessionEvents:495(226596156ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 226364623ns max# 13939207ns}}}
2025-11-11T17:21:43.672178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:43.681695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519262681819442:2048] [node 1] ICP22 created new session: [3:7571519262681819538:2048]
2025-11-11T17:21:43.681776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519262681819538:2048] [node 1] ICS09 handshake done sender: [3:7571519262681819527:2117] self: [3:7571519262681819530:0] peer: [1:7571519264686189320:0] socket: 136 qp: -1
2025-11-11T17:21:43.681836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519262681819538:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:43.681889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:43.879844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:43.682173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519262681819442:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24906583ns) {HandleHandshakeStatus:329(24904048ns) {DropIncomingHandshake:441(2139ns) DropOutgoingHandshake:456(439ns) SwitchToState:178(8335ns) ProcessPendingSessionEvents:495(1038ns)}}
2025-11-11T17:21:43.733348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519262681819540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6478
2025-11-11T17:21:43.733413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519262681819541:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6478
2025-11-11T17:21:43.766422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519262681819522:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.766467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519262681819522:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:43.803767Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519262681819522:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:43.815427Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519262681819541:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:43.821707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519262681819507:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:43.842080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:43.852011Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 4] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# Connection reset by peer state# ReceiveExRequestHeader processed# 0 remain# 8
2025-11-11T17:21:43.852137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519262681819506:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:43.852530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519262681819507:2108] [node 5] ICP22 created new session: [3:7571519262681819543:2108]
2025-11-11T17:21:43.857353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519262681819543:2108] [node 5] ICS09 handshake done sender: [3:7571519262681819522:2116] self: [3:7571519262681819515:0] peer: [5:7571519262338214104:0] socket: 167 qp: -1
2025-11-11T17:21:43.857659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519262681819543:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:43.860655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:43.862124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519262681819507:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(39622205ns) {HandleHandshakeStatus:329(39617125ns) {DropIncomingHandshake:441(526ns) DropOutgoingHandshake:456(865ns) SwitchToState:178(604ns) ProcessPendingSessionEvents:495(503156ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 445595ns max# 47518ns}}}
2025-11-11T17:21:43.862215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519262681819507:2108] [node 5] ICP18 handshake (actor [3:7571519262681819541:2121]) is too old
2025-11-11T17:21:43.879895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519263119004080:2048] [node 1] ICP22 created new session: [4:7571519263119004183:2048]
2025-11-11T17:21:43.879940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519263119004183:2048] [node 1] ICS09 handshake done sender: [4:7571519263119004172:2121] self: [4:7571519263119004179:0] peer: [1:7571519264686189330:0] socket: 131 qp: -1
2025-11-11T17:21:43.879978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519263119004183:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:43.880011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:43.885608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519262338214007:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(49693022ns) {HandleHandshakeStatus:329(49689428ns) {DropIncomingHandshake:441(1529ns) DropOutgoingHandshake:456(10553ns) SwitchToState:178(741ns) ProcessPendingSessionEvents:495(14354280ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12430317ns max# 5195566ns}}}
2025-11-11T17:21:43.965167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:43.965451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519262681819538:2048] [node 1] ICS01 socket: 136 reason# EndOfStream
2025-11-11T17:21:43.965490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519262681819442:2048] [node 1] ICP30 unregister session Session# [3:7571519262681819538:2048] VirtualId# [3:7571519262681819530:0]
2025-11-11T17:21:43.965527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519262681819538:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:43.965626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:43.965709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:43.967290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:43.967528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519265239784980:2048] [node 1] ICS07 socket disconnect 138 reason# EndOfStream
2025-11-11T17:21:43.967561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519265239784980:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:43.967653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:43.967700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519265239784980:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:43.970814Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address
2025-11-11T17:21:43.971143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519265239784879:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:43.971207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519265239784980:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:43.971243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519265239784879:2048] [node 1] ICP30 unregister session Session# [2:7571519265239784980:2048] VirtualId# [2:7571519265239784968:0]
2025-11-11T17:21:43.971345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:43.971395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571519265239784879:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:43.971410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:43.976050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:43.975492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:43.976089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:43.988799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519262338214105:2048] [node 1] ICS07 socket disconnect 156 reason# EndOfStream
2025-11-11T17:21:43.988853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519262338214105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:43.988925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:43.988967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519262338214105:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:43.987732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519263119004183:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-11T17:21:43.987415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571519265239784879:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:43.987782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519263119004080:2048] [node 1] ICP30 unregister session Session# [4:7571519263119004183:2048] VirtualId# [4:7571519263119004179:0]
2025-11-11T17:21:43.987819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519263119004183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:43.987954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:43.988002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:43.990304Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address
2025-11-11T17:21:43.989831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519264897926342:2048] [node 1] ICS07 socket disconnect 137 reason# EndOfStream
2025-11-11T17:21:43.989877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519264897926342:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:43.989935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:43.989972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519264897926342:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:43.993994Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address
2025-11-11T17:21:44.006751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519264897926251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:44.017079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519264897926342:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:44.017128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519264897926251:2048] [node 1] ICP30 unregister session Session# [6:7571519264897926342:2048] VirtualId# [6:7571519264897926334:0]
2025-11-11T17:21:44.017258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:44.017295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519264897926251:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.017311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.017560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [6:7571519264897926251:2048] [node 1] ICP35 event processing took too much time StateWork:293(23041796ns) {HandleHandshakeStatus:395(23027288ns) {LogHandshakeFail:472(8755ns) {UpdateErrorStateLog:544(2630ns)} DropOutgoingHandshake:456(2020ns) ProcessPendingSessionEvents:495(440ns) UnregisterSession:538(49475ns) {ProcessPendingSessionEvents:495(175ns) SwitchToInitialState:201(1969ns) {SwitchToState:178(868ns)}} TransitToErrorState:752(55858ns) {SwitchToState:178(6729ns) ProcessPendingSessionEvents:495(94ns) DropHandshakes:469(2423ns) {DropIncomingHandshake:441(761ns) DropOutgoingHandshake:456(317ns)}}}}
2025-11-11T17:21:44.015044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519262338214005:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:44.018045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519262338214105:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:44.018082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519262338214005:2048] [node 1] ICP30 unregister session Session# [5:7571519262338214105:2048] VirtualId# [5:7571519262338214099:0]
2025-11-11T17:21:44.023776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:44.023824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519262338214005:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.023839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.023612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519264897926251:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:44.024135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [5:7571519262338214005:2048] [node 1] ICP35 event processing took too much time StateWork:293(33365664ns) {HandleHandshakeStatus:395(30170956ns) {LogHandshakeFail:472(6956757ns) {UpdateErrorStateLog:544(7276ns)} DropOutgoingHandshake:456(3131ns) ProcessPendingSessionEvents:495(834ns) UnregisterSession:538(37628ns) {ProcessPendingSessionEvents:495(57ns) SwitchToInitialState:201(769ns) {SwitchToState:178(446ns)}} TransitToErrorState:752(57983ns) {SwitchToState:178(5667ns) ProcessPendingSessionEvents:495(386ns) DropHandshakes:469(3185ns) {DropIncomingHandshake:441(880ns) DropOutgoingHandshake:456(638ns)}}}}
2025-11-11T17:21:44.034364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519262338214005:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:44.096618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:44.096848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519262681819534:2049] [node 2] ICS01 socket: 125 reason# EndOfStream
2025-11-11T17:21:44.096887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519262681819443:2049] [node 2] ICP30 unregister session Session# [3:7571519262681819534:2049] VirtualId# [3:7571519262681819524:0]
2025-11-11T17:21:44.096924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519262681819534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.097022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:44.097069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:44.100035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:44.100274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519263119004171:2049] [node 2] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-11T17:21:44.100309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519263119004171:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.100388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:44.100088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:44.100433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519263119004171:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:44.100315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519262338214102:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-11T17:21:44.100371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519262338214006:2049] [node 2] ICP30 unregister session Session# [5:7571519262338214102:2049] VirtualId# [5:7571519262338214089:0]
2025-11-11T17:21:44.100408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519262338214102:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.100524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:44.100582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:44.101220Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Couldn't connect to any resolved address
2025-11-11T17:21:44.101553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519263119004081:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:44.101637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519263119004171:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:44.101681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519263119004081:2049] [node 2] ICP30 unregister session Session# [4:7571519263119004171:2049] VirtualId# [4:7571519263119004162:0]
2025-11-11T17:21:44.101728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:44.101765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519263119004081:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.101779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.105449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:44.105700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519264897926349:2049] [node 2] ICS01 socket: 169 reason# EndOfStream
2025-11-11T17:21:44.105739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519264897926252:2049] [node 2] ICP30 unregister session Session# [6:7571519264897926349:2049] VirtualId# [6:7571519264897926347:0]
2025-11-11T17:21:44.105775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519264897926349:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.105861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:44.105923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:44.111130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519263119004081:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:44.177967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:44.178184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519264897926338:2050] [node 3] ICS07 socket disconnect 129 reason# EndOfStream
2025-11-11T17:21:44.178213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519264897926338:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.178301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:44.178342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519264897926338:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:44.177840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:44.178100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519263119004177:2050] [node 3] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-11T17:21:44.178129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519263119004177:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.178223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:44.178270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519263119004177:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:44.180043Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address
2025-11-11T17:21:44.180524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519264897926253:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:44.183759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519264897926338:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:44.183803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519264897926253:2050] [node 3] ICP30 unregister session Session# [6:7571519264897926338:2050] VirtualId# [6:7571519264897926330:0]
2025-11-11T17:21:44.183913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:44.183950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519264897926253:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.183967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.178881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:44.179076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519262338214109:2050] [node 3] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-11T17:21:44.179103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519262338214109:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.179171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:44.179220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519262338214109:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:44.180040Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address
2025-11-11T17:21:44.183249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519262338214007:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:44.183317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519262338214109:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:44.183357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519262338214007:2050] [node 3] ICP30 unregister session Session# [5:7571519262338214109:2050] VirtualId# [5:7571519262338214104:0]
2025-11-11T17:21:44.183469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:44.183507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519262338214007:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.183522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.184359Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address
2025-11-11T17:21:44.184590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519263119004082:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:44.189425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519263119004177:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:44.191424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519263119004082:2050] [node 3] ICP30 unregister session Session# [4:7571519263119004177:2050] VirtualId# [4:7571519263119004169:0]
2025-11-11T17:21:44.191507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:44.191868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519263119004082:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.191894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19993) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:44.194291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519264897926253:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:44.196574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519262338214007:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:44.251217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:44.251489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519264897926353:2091] [node 4] ICS01 socket: 159 reason# EndOfStream
2025-11-11T17:21:44.251530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519264897926299:2091] [node 4] ICP30 unregister session Session# [6:7571519264897926353:2091] VirtualId# [6:7571519264897926351:0]
2025-11-11T17:21:44.251570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519264897926353:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.251658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:44.251704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:44.253741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:44.253972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519262338214095:2108] [node 4] ICS01 socket: 121 reason# EndOfStream
2025-11-11T17:21:44.254011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519262338214069:2108] [node 4] ICP30 unregister session Session# [5:7571519262338214095:2108] VirtualId# [5:7571519262338214087:0]
2025-11-11T17:21:44.254045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519262338214095:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.254128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:44.254177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:44.329850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:44.330104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519264897926344:2093] [node 5] ICS01 socket: 143 reason# EndOfStream
2025-11-11T17:21:44.330169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519264897926303:2093] [node 5] ICP30 unregister session Session# [6:7571519264897926344:2093] VirtualId# [6:7571519264897926337:0]
2025-11-11T17:21:44.330264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519264897926344:2093] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:44.330435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:44.330492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:44.424186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519266717152067:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:44.424204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519266717152066:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:44.443248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519266341856333:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:44.443287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519266341856334:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:44.463753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519266417002764:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:44.463859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519266417002765:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:44.484561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519268459909099:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:44.484604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519268459909100:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:44.484645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519268459909101:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:44.509729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519266887721617:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:44.509790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519266887721619:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:44.510680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519266887721618:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:44.530212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519269510142358:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:44.530265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519269510142359:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:44.530328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519269510142360:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:44.598230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519269510142415:2098] [node 5] ICP01 ready to work
2025-11-11T17:21:44.740518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519268459909162:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:44.724093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519266341856396:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:44.739932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519266717152156:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:44.740780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519266717152158:2126] [node 6] ICP01 ready to work
2025-11-11T17:21:44.751010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519266717152157:2116] [node 5] ICP01 ready to work
2025-11-11T17:21:44.739812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519266417002829:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:44.765879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519268459909163:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:44.740262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519266341856397:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:44.755942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519266417002828:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:44.760692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519266417002830:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:44.777808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519266887721680:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:44.778591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519266887721681:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:44.754369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519269510142420:2103] [node 4] ICP01 ready to work
2025-11-11T17:21:44.750124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519266341856398:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:44.822438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519266417002765:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(108799557ns) {Configure:104(108792892ns) {ProcessConfigured:135(5331286ns) {StartInitialHandshake:157(5328325ns) {DropHandshakes:469(3773ns) {DropIncomingHandshake:441(786ns) DropOutgoingHandshake:456(603ns)} GenerateSessionVirtualId:988(1253ns) PrepareNewSessionHandshake:476(4363ns) {SwitchToState:178(1776ns)}}}}}
2025-11-11T17:21:44.826085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519266717152156:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(71728270ns) {Configure:104(71718643ns) {ProcessConfigured:135(41062ns) {StartInitialHandshake:157(39767ns) {DropHandshakes:469(1142ns) {DropIncomingHandshake:441(269ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(338ns) PrepareNewSessionHandshake:476(5016ns) {SwitchToState:178(3721ns)}}}}}
2025-11-11T17:21:44.845751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519266887721680:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21552402ns) {Configure:104(21546166ns) {ProcessConfigured:135(773039ns) {StartInitialHandshake:157(771268ns) {DropHandshakes:469(5214ns) {DropIncomingHandshake:441(544ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(793ns) PrepareNewSessionHandshake:476(1341ns) {SwitchToState:178(677ns)}}}}}
2025-11-11T17:21:44.831522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519266341856333:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(228874820ns) {Configure:104(228868712ns) {ProcessConfigured:135(2788932ns) {StartInitialHandshake:157(2785977ns) {DropHandshakes:469(3789ns) {DropIncomingHandshake:441(811ns) DropOutgoingHandshake:456(566ns)} GenerateSessionVirtualId:988(994ns) PrepareNewSessionHandshake:476(3923ns) {SwitchToState:178(1873ns)}}}}}
2025-11-11T17:21:44.855602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519268459909099:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33397407ns) {Configure:104(32909248ns) {ProcessConfigured:135(65995ns) {StartInitialHandshake:157(63751ns) {DropHandshakes:469(5525ns) {DropIncomingHandshake:441(427ns) DropOutgoingHandshake:456(553ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:476(1094ns) {SwitchToState:178(306ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.841668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519269510142358:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19587036ns) {Configure:104(19580023ns) {ProcessConfigured:135(68861ns) {StartInitialHandshake:157(67033ns) {DropHandshakes:469(5157ns) {DropIncomingHandshake:441(413ns) DropOutgoingHandshake:456(560ns)} GenerateSessionVirtualId:988(716ns) PrepareNewSessionHandshake:476(1582ns) {SwitchToState:178(576ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.854531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519269510142415:2098] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(57366324ns) {Configure:104(57358387ns) {ProcessConfigured:135(11132331ns) {StartInitialHandshake:157(11119266ns) {DropHandshakes:469(2157ns) {DropIncomingHandshake:441(380ns) DropOutgoingHandshake:456(417ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:476(4141ns) {SwitchToState:178(1657ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.865578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519266717152158:2126] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(83869338ns) {Configure:104(83861797ns) {ProcessConfigured:135(13488439ns) {StartInitialHandshake:157(13481971ns) {DropHandshakes:469(5788021ns) {DropIncomingHandshake:441(3392ns) DropOutgoingHandshake:456(412ns)} GenerateSessionVirtualId:988(2919ns) PrepareNewSessionHandshake:476(4373ns) {SwitchToState:178(1894ns)}}}}}
2025-11-11T17:21:44.866330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519266887721684:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:44.884779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519266417002828:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(72026372ns) {Configure:104(72021445ns) {ProcessConfigured:135(12578450ns) {StartInitialHandshake:157(12576624ns) {DropHandshakes:469(1166ns) {DropIncomingHandshake:441(243ns) DropOutgoingHandshake:456(223ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:476(3881ns) {SwitchToState:178(1959ns)}}}}}
2025-11-11T17:21:44.897709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519266887721688:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:44.889930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519268459909173:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:44.909505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519266717152066:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(86303813ns) {Configure:104(86297301ns) {ProcessConfigured:135(65748ns) {StartInitialHandshake:157(63673ns) {DropHandshakes:469(5016ns) {DropIncomingHandshake:441(1048ns) DropOutgoingHandshake:456(339ns)} GenerateSessionVirtualId:988(774ns) PrepareNewSessionHandshake:476(1749ns) {SwitchToState:178(477ns)}}}}}
2025-11-11T17:21:44.910097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519269510142360:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(95232973ns) {Configure:104(95224394ns) {ProcessConfigured:135(27014291ns) {StartInitialHandshake:157(27010780ns) {DropHandshakes:469(2552ns) {DropIncomingHandshake:441(586ns) DropOutgoingHandshake:456(488ns)} GenerateSessionVirtualId:988(1082ns) PrepareNewSessionHandshake:476(5961994ns) {SwitchToState:178(4812ns)}}}}}
2025-11-11T17:21:44.918761Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152166:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.908600Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519268459909162:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(89233185ns) {Configure:104(89227097ns) {ProcessConfigured:135(33814505ns) {StartInitialHandshake:157(33810662ns) {DropHandshakes:469(2466ns) {DropIncomingHandshake:441(442ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(1292ns) PrepareNewSessionHandshake:476(3543ns) {SwitchToState:178(2038ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.897923Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519266887721684:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.918813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519266717152166:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:44.930119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519266341856398:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18640972ns) {Configure:104(18635142ns) {ProcessConfigured:135(59760ns) {StartInitialHandshake:157(57182ns) {DropHandshakes:469(3477ns) {DropIncomingHandshake:441(754ns) DropOutgoingHandshake:456(771ns)} GenerateSessionVirtualId:988(756ns) PrepareNewSessionHandshake:476(2568ns) {SwitchToState:178(1031ns)}}}}}
2025-11-11T17:21:44.923017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519269510142433:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.909082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519268459909101:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(94686349ns) {Configure:104(94677089ns) {ProcessConfigured:135(15055123ns) {StartInitialHandshake:157(15042541ns) {DropHandshakes:469(2542ns) {DropIncomingHandshake:441(599ns) DropOutgoingHandshake:456(441ns)} GenerateSessionVirtualId:988(974ns) PrepareNewSessionHandshake:476(6666ns) {SwitchToState:178(3145ns)}}}}}
2025-11-11T17:21:44.911507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519266887721618:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(94590792ns) {Configure:104(94584888ns) {ProcessConfigured:135(28002415ns) {StartInitialHandshake:157(27999029ns) {DropHandshakes:469(2392ns) {DropIncomingHandshake:441(651ns) DropOutgoingHandshake:456(401ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:476(5054ns) {SwitchToState:178(2347ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.927216Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519266717152166:2139] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.939191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519269510142420:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(69843974ns) {Configure:104(69839440ns) {ProcessConfigured:135(17680147ns) {StartInitialHandshake:157(17677124ns) {DropHandshakes:469(2926ns) {DropIncomingHandshake:441(623ns) DropOutgoingHandshake:456(779ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:476(1252883ns) {SwitchToState:178(3393ns)}}}}}
2025-11-11T17:21:44.943008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519269510142359:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(102744761ns) {Configure:104(102735894ns) {ProcessConfigured:135(5821809ns) {StartInitialHandshake:157(5819603ns) {DropHandshakes:469(2959ns) {DropIncomingHandshake:441(753ns) DropOutgoingHandshake:456(478ns)} GenerateSessionVirtualId:988(840ns) PrepareNewSessionHandshake:476(1528ns) {SwitchToState:178(172ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.919832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519266887721619:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(77499162ns) {Configure:104(77481894ns) {ProcessConfigured:135(15183905ns) {StartInitialHandshake:157(15180682ns) {DropHandshakes:469(2985ns) {DropIncomingHandshake:441(1116ns) DropOutgoingHandshake:456(454ns)} GenerateSessionVirtualId:988(663ns) PrepareNewSessionHandshake:476(4865ns) {SwitchToState:178(2097ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.952073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519266417002764:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(120442749ns) {Configure:104(120433914ns) {ProcessConfigured:135(7443145ns) {StartInitialHandshake:157(7440809ns) {DropHandshakes:469(3373ns) {DropIncomingHandshake:441(839ns) DropOutgoingHandshake:456(744ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:476(4171ns) {SwitchToState:178(1951ns)}}}}}
2025-11-11T17:21:44.910179Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519268459909173:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.944805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519266341856401:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11772
2025-11-11T17:21:44.951231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519266717152172:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54385
2025-11-11T17:21:44.952199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519266417002833:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33138
2025-11-11T17:21:44.951080Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519269510142433:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.945129Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519266887721688:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.954238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519266341856396:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(119159266ns) {Configure:104(119151259ns) {ProcessConfigured:135(19556465ns) {StartInitialHandshake:157(19553854ns) {DropHandshakes:469(2442ns) {DropIncomingHandshake:441(480ns) DropOutgoingHandshake:456(385ns)} GenerateSessionVirtualId:988(675ns) PrepareNewSessionHandshake:476(5937ns) {SwitchToState:178(3052ns)}}}}}
2025-11-11T17:21:44.918280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519268459909099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:44.952684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519266417002838:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33138
2025-11-11T17:21:44.925313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519268459909181:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:44.945017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519268459909180:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:44.951673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519268459909178:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:44.978766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519266341856334:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(130191360ns) {Configure:104(129201048ns) {ProcessConfigured:135(13680615ns) {StartInitialHandshake:157(13676735ns) {DropHandshakes:469(3929071ns) {DropIncomingHandshake:441(1699ns) DropOutgoingHandshake:456(459ns)} GenerateSessionVirtualId:988(1281ns) PrepareNewSessionHandshake:476(4075ns) {SwitchToState:178(1503ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.965723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519269510142415:2098] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:44.957617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519266887721680:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:44.962085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519266717152174:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54385
2025-11-11T17:21:44.966084Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519266417002833:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.987133Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519266341856401:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.971590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519266887721617:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(144008773ns) {Configure:104(144001279ns) {ProcessConfigured:135(5527035ns) {StartInitialHandshake:157(5522884ns) {DropHandshakes:469(1980ns) {DropIncomingHandshake:441(422ns) DropOutgoingHandshake:456(644ns)} GenerateSessionVirtualId:988(682ns) PrepareNewSessionHandshake:476(1757ns) {SwitchToState:178(529ns)}}}}}
2025-11-11T17:21:44.982189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519269510142438:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:44.969082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519266417002829:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(34868218ns) {Configure:104(34860588ns) {ProcessConfigured:135(3114480ns) {StartInitialHandshake:157(3110721ns) {DropHandshakes:469(3791ns) {DropIncomingHandshake:441(797ns) DropOutgoingHandshake:456(651ns)} GenerateSessionVirtualId:988(751ns) PrepareNewSessionHandshake:476(3541ns) {SwitchToState:178(1488ns)}}}}}
2025-11-11T17:21:44.966787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519266717152156:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:44.986768Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519268459909178:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.987445Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519268459909167:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.978875Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519266887721683:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.969672Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152160:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.969180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519266417002764:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:44.978918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519266887721683:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.987474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519268459909167:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:44.969701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519266717152160:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:44.970805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519266417002846:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33138
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.019277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519270636823707:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11772
2025-11-11T17:21:44.981042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519266887721617:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.017808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519269510142438:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.038202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519269510142358:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.987682Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519268459909173:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:44.972224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.012896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519271182688997:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:44.989149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519266417002837:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33138
2025-11-11T17:21:44.972281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519266717152156:2107] [node 4] ICP22 created new session: [1:7571519266717152175:2107]
2025-11-11T17:21:44.980663Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519266717152160:2137] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:44.981882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519266717152177:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54385
2025-11-11T17:21:44.984968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519266717152175:2107] [node 4] ICS09 handshake done sender: [1:7571519266717152166:2139] self: [1:7571519266717152165:0] peer: [4:7571519268459909179:0] socket: 120 qp: -1
2025-11-11T17:21:45.025689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519266341856407:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11772
2025-11-11T17:21:44.990524Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519268459909181:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.007237Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519266417002838:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.985276Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152174:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.014204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519271182688998:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:45.055369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109737:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.009762Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519266417002833:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:45.011649Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519266417002837:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.041244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519266417002764:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.043306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519266417002830:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(77123479ns) {Configure:104(77116588ns) {ProcessConfigured:135(7939788ns) {StartInitialHandshake:157(7935735ns) {DropHandshakes:469(4292ns) {DropIncomingHandshake:441(849ns) DropOutgoingHandshake:456(998ns)} GenerateSessionVirtualId:988(765ns) PrepareNewSessionHandshake:476(4174ns) {SwitchToState:178(1709ns)}}}}}
2025-11-11T17:21:45.046800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519270711970146:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33138
2025-11-11T17:21:45.053070Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519266417002835:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.053103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519266417002835:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.056381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:45.056433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519266417002764:2048] [node 1] ICP22 created new session: [3:7571519270711970149:2048]
2025-11-11T17:21:45.070152Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519270711970146:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.070517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519266417002765:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.070620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519270711970151:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33138
2025-11-11T17:21:45.072841Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519266417002835:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:45.041333Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519270636823707:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.084199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.081674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519270711970149:2048] [node 1] ICS09 handshake done sender: [3:7571519266417002833:2112] self: [3:7571519266417002845:0] peer: [1:7571519266717152159:0] socket: 119 qp: -1
2025-11-11T17:21:45.084668Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519269510142438:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:45.015582Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519266887721683:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:44.997484Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519268459909167:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:45.095154Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519273805109737:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.095224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519269510142358:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.095309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:45.095375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519269510142358:2048] [node 1] ICP22 created new session: [6:7571519273805109740:2048]
2025-11-11T17:21:45.095420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519273805109740:2048] [node 1] ICS09 handshake done sender: [6:7571519269510142438:2116] self: [6:7571519273805109736:0] peer: [1:7571519266717152168:0] socket: 149 qp: -1
2025-11-11T17:21:45.095454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519273805109740:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:45.095513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:44.987108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519266717152175:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.044135Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519266341856407:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.987196Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152163:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:44.997546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519268459909099:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.016000Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519266887721688:2113] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:45.090413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519270711970149:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:45.098492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519269510142420:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.062080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519266341856333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.098926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:45.115896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109735:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.002432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:45.016124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519271182688999:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:44.987221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519266717152163:2138] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.081125Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519266341856404:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.016187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519266887721684:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:45.017560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519266887721680:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.017626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:45.107549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519266417002828:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.135081Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519269510142433:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:45.006902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519268459909099:2048] [node 1] ICP22 created new session: [4:7571519268459909185:2048]
2025-11-11T17:21:44.987611Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152172:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.081165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519266341856404:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.107624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:45.017427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519268459909185:2048] [node 1] ICS09 handshake done sender: [4:7571519268459909173:2113] self: [4:7571519268459909179:0] peer: [1:7571519266717152165:0] socket: 122 qp: -1
2025-11-11T17:21:45.021582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519266887721681:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.162129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519269510142415:2098] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.085865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519266341856408:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.181681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:45.189091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519269510142415:2098] [node 5] ICP22 created new session: [6:7571519273805109745:2098]
2025-11-11T17:21:44.987626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:45.107689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519266417002828:2108] [node 5] ICP22 created new session: [3:7571519270711970154:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.017477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519268459909185:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:45.021642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:45.085908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519266341856408:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.106061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519270636823712:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11772
2025-11-11T17:21:45.110874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519266341856408:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:45.111998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519266341856334:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.111659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519270711970155:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33138
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:44.997330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519266717152067:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.112054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:45.021689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519266887721681:2109] [node 6] ICP22 created new session: [5:7571519271182689001:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.112101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519266341856334:2049] [node 3] ICP22 created new session: [2:7571519270636823715:2049]
2025-11-11T17:21:45.020315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519268459909163:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.113171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519270711970154:2108] [node 5] ICS09 handshake done sender: [3:7571519266417002835:2113] self: [3:7571519266417002834:0] peer: [5:7571519271182689006:0] socket: 142 qp: -1
2025-11-11T17:21:45.021736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519271182689001:2109] [node 6] ICS09 handshake done sender: [5:7571519266887721683:2110] self: [5:7571519266887721682:0] peer: [6:7571519269510142437:0] socket: 124 qp: -1
2025-11-11T17:21:45.204354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109744:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.020385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:45.112166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519270636823715:2049] [node 3] ICS09 handshake done sender: [2:7571519266341856408:2115] self: [2:7571519266341856406:0] peer: [3:7571519270711970152:0] socket: 163 qp: -1
2025-11-11T17:21:45.000109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519266717152182:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54385
2025-11-11T17:21:45.114403Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519270711970146:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:45.021776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519271182689001:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:45.035451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:45.204516Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14847) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:45.000347Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519266717152163:2138] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:45.036789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519268459909163:2108] [node 5] ICP22 created new session: [4:7571519272754876484:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.119951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519270711970154:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:45.021807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:45.112205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519270636823715:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:45.204584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.042526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519272754876484:2108] [node 5] ICS09 handshake done sender: [4:7571519268459909167:2111] self: [4:7571519268459909166:0] peer: [5:7571519266887721695:0] socket: 118 qp: -1
2025-11-11T17:21:45.023765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519266887721617:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.023821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:45.023864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519266887721617:2048] [node 1] ICP22 created new session: [5:7571519271182689003:2048]
2025-11-11T17:21:45.023912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519271182689003:2048] [node 1] ICS09 handshake done sender: [5:7571519266887721684:2111] self: [5:7571519266887721699:0] peer: [1:7571519266717152162:0] socket: 116 qp: -1
2025-11-11T17:21:45.023944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519271182689003:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:45.023975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:45.040626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519266887721696:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:45.041124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519266887721680:2108] [node 4] ICP22 created new session: [5:7571519271182689000:2108]
2025-11-11T17:21:45.046061Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519266887721696:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.046350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519266887721619:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.047982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519271182689000:2108] [node 4] ICS09 handshake done sender: [5:7571519266887721688:2113] self: [5:7571519266887721695:0] peer: [4:7571519268459909166:0] socket: 123 qp: -1
2025-11-11T17:21:45.048037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519271182689000:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:45.048083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:45.054154Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519271182688999:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.054520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519266887721618:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.054606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519271182689008:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:45.079864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519266887721680:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(61440611ns) {HandleHandshakeStatus:329(61437791ns) {DropIncomingHandshake:441(970ns) DropOutgoingHandshake:456(5326ns) SwitchToState:178(516ns) ProcessPendingSessionEvents:495(12804817ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12736394ns max# 12299460ns}}}
2025-11-11T17:21:45.107161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519266887721696:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:45.115935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519266887721619:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.115983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:45.119907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519266887721619:2050] [node 3] ICP22 created new session: [5:7571519271182689010:2050]
2025-11-11T17:21:45.141864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519271182689010:2050] [node 3] ICS09 handshake done sender: [5:7571519266887721696:2116] self: [5:7571519271182689006:0] peer: [3:7571519266417002834:0] socket: 143 qp: -1
2025-11-11T17:21:45.153907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519271182689010:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:45.165864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:45.222074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519271182689011:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57154
2025-11-11T17:21:45.222534Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519271182688999:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:45.222723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519266887721618:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.227623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:45.229117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519266887721619:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(71891731ns) {HandleHandshakeStatus:329(71888716ns) {DropIncomingHandshake:441(1024ns) DropOutgoingHandshake:456(4956ns) SwitchToState:178(637ns) ProcessPendingSessionEvents:495(7218057ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7149288ns max# 6707227ns}}}
2025-11-11T17:21:45.231002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519266887721618:2049] [node 2] ICP22 created new session: [5:7571519271182689013:2049]
2025-11-11T17:21:45.238445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519271182689013:2049] [node 2] ICS09 handshake done sender: [5:7571519271182688999:2120] self: [5:7571519271182689009:0] peer: [2:7571519266341856403:0] socket: 161 qp: -1
2025-11-11T17:21:45.243017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519271182689013:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:45.243890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519266887721617:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(192071148ns) {HandleHandshakeStatus:329(192068863ns) {DropIncomingHandshake:441(635ns) DropOutgoingHandshake:456(9132ns) SwitchToState:178(283ns) ProcessPendingSessionEvents:495(163164016ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 147915470ns max# 17925369ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.000643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519266717152157:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.122192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:45.112243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:45.113267Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519266341856404:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:45.187048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519266341856334:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(74217999ns) {HandleHandshakeStatus:329(74213314ns) {DropIncomingHandshake:441(14199ns) DropOutgoingHandshake:456(561ns) SwitchToState:178(400ns) ProcessPendingSessionEvents:495(73828833ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67039517ns max# 10822792ns}}}
2025-11-11T17:21:45.193145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519270636823710:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11772
2025-11-11T17:21:45.201738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519266341856398:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.201789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:45.201842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519266341856398:2108] [node 5] ICP22 created new session: [2:7571519270636823719:2108]
2025-11-11T17:21:45.201894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519270636823719:2108] [node 5] ICS09 handshake done sender: [2:7571519266341856404:2112] self: [2:7571519266341856403:0] peer: [5:7571519271182689009:0] socket: 156 qp: -1
2025-11-11T17:21:45.201937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519270636823719:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:45.202020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:45.203419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519270636823717:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11772
2025-11-11T17:21:45.206090Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519270636823712:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.206270Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519270636823707:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:45.210778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519266341856333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.210853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:45.210911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519266341856333:2048] [node 1] ICP22 created new session: [2:7571519270636823723:2048]
2025-11-11T17:21:45.211000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519270636823723:2048] [node 1] ICS09 handshake done sender: [2:7571519270636823707:2116] self: [2:7571519270636823711:0] peer: [1:7571519266717152170:0] socket: 159 qp: -1
2025-11-11T17:21:45.211042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519270636823723:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:45.211086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:45.213580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519266341856397:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(164342360ns) {Configure:104(164337695ns) {ProcessConfigured:135(7971778ns) {StartInitialHandshake:157(7968225ns) {DropHandshakes:469(3765ns) {DropIncomingHandshake:441(879ns) DropOutgoingHandshake:456(744ns)} GenerateSessionVirtualId:988(1431ns) PrepareNewSessionHandshake:476(1677ns) {SwitchToState:178(456ns)}}}}}
2025-11-11T17:21:45.224577Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519270636823710:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.227926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519266341856398:2108] [node 5] ICP18 handshake (actor [2:7571519270636823710:2117]) is too old
2025-11-11T17:21:45.241756Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519270636823722:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.241790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519270636823722:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.245861Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519270636823722:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:45.000688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:45.042576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519272754876484:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:45.250378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:45.206544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519273805109745:2098] [node 5] ICS09 handshake done sender: [6:7571519269510142433:2115] self: [6:7571519269510142437:0] peer: [5:7571519266887721682:0] socket: 133 qp: -1
2025-11-11T17:21:45.136477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519266417002765:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.042630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:45.083042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519268459909099:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(84688222ns) {HandleHandshakeStatus:329(84685400ns) {DropIncomingHandshake:441(471ns) DropOutgoingHandshake:456(4830922ns) SwitchToState:178(1438ns) ProcessPendingSessionEvents:495(25678744ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25619714ns max# 14537213ns}}}
2025-11-11T17:21:45.123735Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519268459909174:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.123787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519268459909174:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.134644Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519268459909174:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:45.166384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519268459909162:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.166454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:45.166510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519268459909162:2109] [node 6] ICP22 created new session: [4:7571519272754876489:2109]
2025-11-11T17:21:45.166570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519272754876489:2109] [node 6] ICS09 handshake done sender: [4:7571519268459909174:2114] self: [4:7571519268459909170:0] peer: [6:7571519273805109742:0] socket: 129 qp: -1
2025-11-11T17:21:45.166613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519272754876489:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:45.166647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:45.178560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519268459909163:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(157412297ns) {HandleHandshakeStatus:329(157408235ns) {DropIncomingHandshake:441(13997ns) DropOutgoingHandshake:456(665ns) SwitchToState:178(12909ns) ProcessPendingSessionEvents:495(134973533ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 124859367ns max# 15678418ns}}}
2025-11-11T17:21:45.218001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519272754876491:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:45.223337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519272754876487:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:45.239992Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519272754876487:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.240045Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519272754876491:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.241202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519268459909101:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.241278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519268459909100:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.249738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519272754876494:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:45.254312Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519272754876491:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:45.258109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519266341856397:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.206595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519273805109745:2098] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.136549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:45.000734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519266717152157:2116] [node 5] ICP22 created new session: [1:7571519266717152185:2116]
2025-11-11T17:21:45.264749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519268459909100:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.219670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:45.265380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:45.000787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519266717152185:2116] [node 5] ICS09 handshake done sender: [1:7571519266717152163:2138] self: [1:7571519266717152162:0] peer: [5:7571519266887721699:0] socket: 115 qp: -1
2025-11-11T17:21:45.136611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519266417002765:2049] [node 2] ICP22 created new session: [3:7571519270711970158:2049]
2025-11-11T17:21:45.267534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519266341856397:2107] [node 4] ICP22 created new session: [2:7571519270636823729:2107]
2025-11-11T17:21:45.264816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:45.223299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519269510142420:2103] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14847) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519273805109737:2118] held: no
2025-11-11T17:21:45.136660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519270711970158:2049] [node 2] ICS09 handshake done sender: [3:7571519270711970146:2119] self: [3:7571519270711970152:0] peer: [2:7571519266341856406:0] socket: 125 qp: -1
2025-11-11T17:21:45.000824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519266717152185:2116] [node 5] ICS10 traffic start
2025-11-11T17:21:45.268536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519270636823729:2107] [node 4] ICS09 handshake done sender: [2:7571519270636823722:2120] self: [2:7571519270636823718:0] peer: [4:7571519272754876493:0] socket: 140 qp: -1
2025-11-11T17:21:45.265251Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:45.227541Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519273805109739:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.000864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:45.136695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519270711970158:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:45.001899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:45.011501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519266717152067:2049] [node 3] ICP22 created new session: [1:7571519266717152181:2049]
2025-11-11T17:21:45.015387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519266717152181:2049] [node 3] ICS09 handshake done sender: [1:7571519266717152160:2137] self: [1:7571519266717152159:0] peer: [3:7571519266417002845:0] socket: 114 qp: -1
2025-11-11T17:21:45.016090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519266717152181:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:45.016141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:45.021140Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152177:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.035480Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152182:2145] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.056396Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152169:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.056435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519266717152169:2140] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.063925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519266717152156:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(97399646ns) {HandleHandshakeStatus:329(96342700ns) {DropIncomingHandshake:441(404ns) DropOutgoingHandshake:456(427ns) SwitchToState:178(528ns) ProcessPendingSessionEvents:495(65423074ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 65310798ns max# 17261834ns}}}
2025-11-11T17:21:45.064179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519266717152156:2107] [node 4] ICP18 handshake (actor [1:7571519266717152172:2142]) is too old
2025-11-11T17:21:45.072601Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519266717152169:2140] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:45.109892Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519266717152171:2141] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.109921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519266717152171:2141] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.110522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519266717152158:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.110577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:45.110621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519266717152158:2126] [node 6] ICP22 created new session: [1:7571519271012119486:2126]
2025-11-11T17:21:45.110672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519271012119486:2126] [node 6] ICS09 handshake done sender: [1:7571519266717152169:2140] self: [1:7571519266717152168:0] peer: [6:7571519273805109736:0] socket: 148 qp: -1
2025-11-11T17:21:45.116028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519271012119486:2126] [node 6] ICS10 traffic start
2025-11-11T17:21:45.120851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:45.170231Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519266717152171:2141] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:45.193905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519266717152066:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.193982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:45.206533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519266717152066:2048] [node 2] ICP22 created new session: [1:7571519271012119489:2048]
2025-11-11T17:21:45.213231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519271012119489:2048] [node 2] ICS09 handshake done sender: [1:7571519266717152171:2141] self: [1:7571519266717152170:0] peer: [2:7571519270636823711:0] socket: 155 qp: -1
2025-11-11T17:21:45.217046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519271012119489:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:45.217097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:45.283781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519266717152067:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(269842442ns) {HandleHandshakeStatus:329(269840638ns) {DropIncomingHandshake:441(325ns) DropOutgoingHandshake:456(919ns) SwitchToState:178(269ns) ProcessPendingSessionEvents:495(224709404ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 219472652ns max# 13178066ns}}}
2025-11-11T17:21:45.284546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519266717152067:2049] [node 3] ICP18 handshake (actor [1:7571519266717152177:2144]) is too old
2025-11-11T17:21:45.265482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519268459909100:2049] [node 2] ICP22 created new session: [4:7571519272754876495:2049]
2025-11-11T17:21:45.268572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519270636823729:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:45.243113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519269510142360:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.136733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.268182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519272754876495:2049] [node 2] ICS09 handshake done sender: [4:7571519272754876491:2120] self: [4:7571519272754876493:0] peer: [2:7571519270636823718:0] socket: 141 qp: -1
2025-11-11T17:21:45.247161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109747:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.268818Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519266341856405:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.143077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519266417002764:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(115912078ns) {HandleHandshakeStatus:329(111354470ns) {DropIncomingHandshake:441(2402ns) DropOutgoingHandshake:456(6727ns) SwitchToState:178(725ns) ProcessPendingSessionEvents:495(16413042ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16332283ns max# 15943315ns}}}
2025-11-11T17:21:45.183090Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519270711970151:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.241808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519270711970147:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.241835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519270711970147:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.256192Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519266417002844:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.256272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519266417002844:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.264270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519266417002828:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(118833724ns) {HandleHandshakeStatus:329(118829599ns) {DropIncomingHandshake:441(20491ns) DropOutgoingHandshake:456(631ns) SwitchToState:178(831ns) ProcessPendingSessionEvents:495(62376726ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 58452616ns max# 16289499ns}}}
2025-11-11T17:21:45.264557Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519266417002844:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:45.268588Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519270711970147:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:45.279508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519266417002829:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.279596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:45.279643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519266417002829:2109] [node 6] ICP22 created new session: [3:7571519270711970163:2109]
2025-11-11T17:21:45.279696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519270711970163:2109] [node 6] ICS09 handshake done sender: [3:7571519266417002844:2117] self: [3:7571519266417002842:0] peer: [6:7571519273805109748:0] socket: 168 qp: -1
2025-11-11T17:21:45.279735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519270711970163:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:45.279771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:45.302204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519266417002765:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(104000885ns) {HandleHandshakeStatus:329(103997038ns) {DropIncomingHandshake:441(886ns) DropOutgoingHandshake:456(6801ns) SwitchToState:178(1101ns) ProcessPendingSessionEvents:495(103516278ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 95100569ns max# 22919764ns}}}
2025-11-11T17:21:45.310503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519266417002830:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.345782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:45.350716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519266417002830:2107] [node 4] ICP22 created new session: [3:7571519270711970165:2107]
2025-11-11T17:21:45.263866Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519273805109747:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:45.268521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519272754876495:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:45.268841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519266341856405:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:45.266163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109749:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.274302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:45.376073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519270711970165:2107] [node 4] ICS09 handshake done sender: [3:7571519270711970147:2120] self: [3:7571519270711970145:0] peer: [4:7571519272754876492:0] socket: 176 qp: -1
2025-11-11T17:21:45.374876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519266887721618:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(35372963ns) {HandleHandshakeStatus:329(35370183ns) {DropIncomingHandshake:441(970ns) DropOutgoingHandshake:456(4883ns) SwitchToState:178(712ns) ProcessPendingSessionEvents:495(1777041ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1723457ns max# 1318532ns}}}
2025-11-11T17:21:45.382908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519266717152066:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(188039568ns) {HandleHandshakeStatus:329(188034487ns) {DropIncomingHandshake:441(19444ns) DropOutgoingHandshake:456(965ns) SwitchToState:178(1064ns) ProcessPendingSessionEvents:495(156511101ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 146560088ns max# 19776854ns}}}
2025-11-11T17:21:45.270633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:45.376148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519270711970165:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:45.394662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:45.280743Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519266341856405:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:45.310732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519266341856396:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:45.323709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:45.337757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519266341856396:2109] [node 6] ICP22 created new session: [2:7571519270636823732:2109]
2025-11-11T17:21:45.361768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519270636823732:2109] [node 6] ICS09 handshake done sender: [2:7571519266341856405:2113] self: [2:7571519266341856402:0] peer: [6:7571519273805109750:0] socket: 137 qp: -1
2025-11-11T17:21:45.363203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519266341856397:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(104208057ns) {HandleHandshakeStatus:329(104204341ns) {DropIncomingHandshake:441(21609ns) DropOutgoingHandshake:456(625ns) SwitchToState:178(747ns) ProcessPendingSessionEvents:495(60143241ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60071612ns max# 36375161ns}}}
2025-11-11T17:21:45.368027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519270636823732:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:45.374759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:45.268307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519269510142359:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:45.346684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519272754876496:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65337
2025-11-11T17:21:45.349793Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:45.271346Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519273805109739:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:45.278964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519269510142360:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.358470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519268459909101:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519272754876487:2119] held: no
2025-11-11T17:21:45.408007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519268459909101:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(57118295ns) {HandleHandshakeStatus:395(57113231ns) {LogHandshakeFail:472(8328803ns) {UpdateErrorStateLog:544(6010ns)} DropOutgoingHandshake:456(443ns) ProcessPendingSessionEvents:495(48724648ns) {unordered DropSessionEvent:512 num# 100 sum# 10691827ns max# 5577060ns}}}
2025-11-11T17:21:45.305890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:45.305971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519269510142360:2050] [node 3] ICP22 created new session: [6:7571519273805109752:2050]
2025-11-11T17:21:45.306064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519273805109752:2050] [node 3] ICS09 handshake done sender: [6:7571519273805109739:2120] self: [6:7571519273805109748:0] peer: [3:7571519266417002842:0] socket: 171 qp: -1
2025-11-11T17:21:45.306112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519273805109752:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:45.306156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:45.408162Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519272754876487:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:45.334022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519269510142420:2103] [node 4] ICP35 event processing took too much time PendingConnection:278(40397736ns) {HandleHandshakeStatus:395(36722994ns) {LogHandshakeFail:472(3433226ns) {UpdateErrorStateLog:544(9885ns)} DropOutgoingHandshake:456(336ns) ProcessPendingSessionEvents:495(33219365ns) {unordered DropSessionEvent:512 num# 100 sum# 15400208ns max# 6159558ns}}}
2025-11-11T17:21:45.356385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519269510142360:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(76499788ns) {HandleHandshakeStatus:329(76496411ns) {DropIncomingHandshake:441(962ns) DropOutgoingHandshake:456(7154ns) SwitchToState:178(582ns) ProcessPendingSessionEvents:495(49126023ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 49026500ns max# 24184429ns}}}
2025-11-11T17:21:45.356575Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:45.365764Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519273805109737:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:45.391889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519269510142420:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.392006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:45.392077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519269510142420:2103] [node 4] ICP22 created new session: [6:7571519273805109754:2103]
2025-11-11T17:21:45.408500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519268459909100:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(143444011ns) {HandleHandshakeStatus:329(143440678ns) {DropIncomingHandshake:441(1276ns) DropOutgoingHandshake:456(6984ns) SwitchToState:178(567ns) ProcessPendingSessionEvents:495(128447868ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 118101599ns max# 23195791ns}}}
2025-11-11T17:21:45.392128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519273805109754:2103] [node 4] ICS09 handshake done sender: [6:7571519273805109737:2118] self: [6:7571519273805109742:0] peer: [4:7571519268459909170:0] socket: 162 qp: -1
2025-11-11T17:21:45.408685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519268459909100:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:45.409569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519268459909101:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.410172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:45.410539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519268459909101:2050] [node 3] ICP22 created new session: [4:7571519272754876498:2050]
2025-11-11T17:21:45.392177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519273805109754:2103] [node 4] ICS10 traffic start
2025-11-11T17:21:45.410588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519272754876498:2050] [node 3] ICS09 handshake done sender: [4:7571519272754876487:2119] self: [4:7571519272754876492:0] peer: [3:7571519270711970145:0] socket: 177 qp: -1
2025-11-11T17:21:45.392225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:45.410624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519272754876498:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:45.394626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519269510142359:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519273805109747:2122] held: no
2025-11-11T17:21:45.413098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:45.407113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519273805109751:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60753
2025-11-11T17:21:45.408388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519269510142415:2098] [node 5] ICP35 event processing took too much time PendingConnection:278(245308855ns) {HandleHandshakeStatus:329(245304555ns) {DropIncomingHandshake:441(1281ns) DropOutgoingHandshake:456(16789ns) SwitchToState:178(661ns) ProcessPendingSessionEvents:495(178242160ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 163903471ns max# 9573955ns}}}
2025-11-11T17:21:45.409636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519269510142359:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17800096ns) {HandleHandshakeStatus:395(14288721ns) {LogHandshakeFail:472(957640ns) {UpdateErrorStateLog:544(4537ns)} DropOutgoingHandshake:456(421ns) ProcessPendingSessionEvents:495(13267753ns) {unordered DropSessionEvent:512 num# 100 sum# 13230473ns max# 3515645ns}}}
2025-11-11T17:21:45.413358Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519273805109747:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:45.416289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519269510142359:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:45.419814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519266417002830:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(116705099ns) {HandleHandshakeStatus:329(116700177ns) {DropIncomingHandshake:441(25143ns) DropOutgoingHandshake:456(693ns) SwitchToState:178(694ns) ProcessPendingSessionEvents:495(12942074ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12833566ns max# 3081824ns}}}
2025-11-11T17:21:45.421807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519266341856396:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(110239853ns) {HandleHandshakeStatus:329(110236548ns) {DropIncomingHandshake:441(24936ns) DropOutgoingHandshake:456(393ns) SwitchToState:178(5009ns) ProcessPendingSessionEvents:495(27311714ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27212154ns max# 14535679ns}}}
2025-11-11T17:21:45.416342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:45.416397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519269510142359:2049] [node 2] ICP22 created new session: [6:7571519273805109756:2049]
2025-11-11T17:21:45.416441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519273805109756:2049] [node 2] ICS09 handshake done sender: [6:7571519273805109747:2122] self: [6:7571519273805109750:0] peer: [2:7571519266341856402:0] socket: 136 qp: -1
2025-11-11T17:21:45.416475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519273805109756:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:45.416510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:45.445628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:45.445885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519270636823723:2048] [node 1] ICS01 socket: 159 reason# EndOfStream
2025-11-11T17:21:45.445945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519266341856333:2048] [node 1] ICP30 unregister session Session# [2:7571519270636823723:2048] VirtualId# [2:7571519270636823711:0]
2025-11-11T17:21:45.445980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519270636823723:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.446055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:45.446102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:45.452038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:45.453057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519270711970149:2048] [node 1] ICS01 socket: 119 reason# EndOfStream
2025-11-11T17:21:45.453114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519266417002764:2048] [node 1] ICP30 unregister session Session# [3:7571519270711970149:2048] VirtualId# [3:7571519266417002845:0]
2025-11-11T17:21:45.453150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519270711970149:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.453246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:45.453321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:45.457080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:45.457315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519273805109740:2048] [node 1] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-11T17:21:45.457358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519273805109740:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.457431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:45.457496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519273805109740:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:45.458206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:45.458056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:45.458451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519271182689003:2048] [node 1] ICS07 socket disconnect 116 reason# EndOfStream
2025-11-11T17:21:45.458628Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address
2025-11-11T17:21:45.458489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519271182689003:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.458825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519269510142358:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.458573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:45.458937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519273805109740:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.458974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519269510142358:2048] [node 1] ICP30 unregister session Session# [6:7571519273805109740:2048] VirtualId# [6:7571519273805109736:0]
2025-11-11T17:21:45.458614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519271182689003:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:45.459023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:45.458405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519268459909185:2048] [node 1] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-11T17:21:45.458443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519268459909185:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.458518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:45.458573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519268459909185:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:45.459065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519269510142358:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.459087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.459413Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address
2025-11-11T17:21:45.459465Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address
2025-11-11T17:21:45.459551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519266887721617:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.459508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519268459909099:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.459643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519271182689003:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.459567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519268459909185:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.459773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519268459909099:2048] [node 1] ICP30 unregister session Session# [4:7571519268459909185:2048] VirtualId# [4:7571519268459909179:0]
2025-11-11T17:21:45.459685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519266887721617:2048] [node 1] ICP30 unregister session Session# [5:7571519271182689003:2048] VirtualId# [5:7571519266887721699:0]
2025-11-11T17:21:45.459785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:45.459831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519266887721617:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.459844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.459957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:45.459993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519268459909099:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.460031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.470079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519266887721617:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:45.470556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519269510142358:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:45.470592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519268459909099:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:45.563015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:45.563308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519270711970158:2049] [node 2] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-11T17:21:45.563342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519270711970158:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.563470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:45.563514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519270711970158:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:45.564399Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address
2025-11-11T17:21:45.566959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:45.569329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519271182689013:2049] [node 2] ICS07 socket disconnect 161 reason# EndOfStream
2025-11-11T17:21:45.569398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519271182689013:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.569523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:45.569567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519271182689013:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:45.570303Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address
2025-11-11T17:21:45.565487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:45.567271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519272754876495:2049] [node 2] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-11T17:21:45.569306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519272754876495:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.570335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:45.570405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519272754876495:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:45.571083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519266887721618:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.575262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519271182689013:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.575309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519266887721618:2049] [node 2] ICP30 unregister session Session# [5:7571519271182689013:2049] VirtualId# [5:7571519271182689009:0]
2025-11-11T17:21:45.575417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:45.575451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519266887721618:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.575464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.564798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519266417002765:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.564931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519270711970158:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.564970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519266417002765:2049] [node 2] ICP30 unregister session Session# [3:7571519270711970158:2049] VirtualId# [3:7571519270711970152:0]
2025-11-11T17:21:45.565081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:45.565151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519266417002765:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.565191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.575889Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address
2025-11-11T17:21:45.579430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519266417002765:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:45.584033Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519268459909100:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.584209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519272754876495:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.581165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:45.581386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519273805109756:2049] [node 2] ICS01 socket: 136 reason# EndOfStream
2025-11-11T17:21:45.581432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519269510142359:2049] [node 2] ICP30 unregister session Session# [6:7571519273805109756:2049] VirtualId# [6:7571519273805109750:0]
2025-11-11T17:21:45.581473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519273805109756:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.581605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:45.581659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:45.584257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519268459909100:2049] [node 2] ICP30 unregister session Session# [4:7571519272754876495:2049] VirtualId# [4:7571519272754876493:0]
2025-11-11T17:21:45.584378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:45.584415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519268459909100:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.584429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.589730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519266887721618:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:45.593912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519268459909100:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:45.637848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:45.638065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519272754876498:2050] [node 3] ICS01 socket: 177 reason# EndOfStream
2025-11-11T17:21:45.638105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519268459909101:2050] [node 3] ICP30 unregister session Session# [4:7571519272754876498:2050] VirtualId# [4:7571519272754876492:0]
2025-11-11T17:21:45.638139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519272754876498:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.638244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:45.638300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:45.639281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:45.639622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519271182689010:2050] [node 3] ICS07 socket disconnect 143 reason# EndOfStream
2025-11-11T17:21:45.639666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519271182689010:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.639803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:45.639898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519271182689010:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:45.637887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:45.638131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519273805109752:2050] [node 3] ICS07 socket disconnect 171 reason# EndOfStream
2025-11-11T17:21:45.638163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519273805109752:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.638287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:45.638341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519273805109752:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:45.646575Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address
2025-11-11T17:21:45.646895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519269510142360:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.646952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519273805109752:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.646983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519269510142360:2050] [node 3] ICP30 unregister session Session# [6:7571519273805109752:2050] VirtualId# [6:7571519273805109748:0]
2025-11-11T17:21:45.647274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:45.647315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519269510142360:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.647329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.647612Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address
2025-11-11T17:21:45.647827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519266887721619:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.647899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519271182689010:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.647929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519266887721619:2050] [node 3] ICP30 unregister session Session# [5:7571519271182689010:2050] VirtualId# [5:7571519271182689006:0]
2025-11-11T17:21:45.648192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:45.648231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519266887721619:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.648244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.657400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519269510142360:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:45.658393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519266887721619:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:45.691929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:45.692124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519271182689000:2108] [node 4] ICS01 socket: 123 reason# EndOfStream
2025-11-11T17:21:45.692156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519266887721680:2108] [node 4] ICP30 unregister session Session# [5:7571519271182689000:2108] VirtualId# [5:7571519266887721695:0]
2025-11-11T17:21:45.692182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519271182689000:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.692254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:45.692323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:45.693126Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:45.693366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519273805109754:2103] [node 4] ICS01 socket: 162 reason# EndOfStream
2025-11-11T17:21:45.693400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519269510142420:2103] [node 4] ICP30 unregister session Session# [6:7571519273805109754:2103] VirtualId# [6:7571519273805109742:0]
2025-11-11T17:21:45.693429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519273805109754:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.693493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:45.693528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:45.747555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:45.747796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519273805109745:2098] [node 5] ICS07 socket disconnect 133 reason# EndOfStream
2025-11-11T17:21:45.747834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519273805109745:2098] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:45.747925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:45.747969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519273805109745:2098] [node 5] ICS15 start handshake
2025-11-11T17:21:45.748996Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17119) Couldn't connect to any resolved address
2025-11-11T17:21:45.749246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519269510142415:2098] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17119) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:45.749295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519273805109745:2098] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:45.749342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519269510142415:2098] [node 5] ICP30 unregister session Session# [6:7571519273805109745:2098] VirtualId# [6:7571519269510142437:0]
2025-11-11T17:21:45.749459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:21:45.749492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519269510142415:2098] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17119) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.749507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17119) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:45.758793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519269510142415:2098] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:45.850340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519272386382558:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:45.850386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519272386382559:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:45.874781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519270621449761:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:45.875063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519270621449762:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:45.901492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519269725106774:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:45.901533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519269725106775:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:45.926152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519270705085187:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:45.926197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519270705085188:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:45.926208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519270705085186:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:45.954924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519273594606016:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:45.954963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519273594606017:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:45.954987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519273594606018:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:45.983575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519272433949779:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:45.983633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519272433949780:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:45.983668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519272433949781:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:46.081204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519276728917137:2105] [node 5] ICP01 ready to work
2025-11-11T17:21:46.152547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519274916417148:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:46.170268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519276681349918:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:46.161418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519274916417147:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:46.173038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519276681349917:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:46.178766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519277889573376:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:46.178835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519274020074135:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:46.178607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519276728917139:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:46.179404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519276681349919:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:46.170363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519275000052546:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:46.179997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519274020074133:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:46.179977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519277889573375:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:46.187034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519274916417149:2126] [node 6] ICP01 ready to work
2025-11-11T17:21:46.180334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519275000052547:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:46.179737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519274020074134:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:46.200681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519272386382559:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19309357ns) {Configure:104(19301745ns) {ProcessConfigured:135(1402743ns) {StartInitialHandshake:157(1399668ns) {DropHandshakes:469(2959ns) {DropIncomingHandshake:441(643ns) DropOutgoingHandshake:456(480ns)} GenerateSessionVirtualId:988(1385ns) PrepareNewSessionHandshake:476(4490ns) {SwitchToState:178(2345ns)}}}}}
2025-11-11T17:21:46.205975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519276681349917:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26915514ns) {Configure:104(26912056ns) {ProcessConfigured:135(4362277ns) {StartInitialHandshake:157(4358921ns) {DropHandshakes:469(1253ns) {DropIncomingHandshake:441(268ns) DropOutgoingHandshake:456(330ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:476(3336ns) {SwitchToState:178(515ns)}}}}}
2025-11-11T17:21:46.200680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519276728917139:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18180077ns) {Configure:104(18176999ns) {ProcessConfigured:135(4659455ns) {StartInitialHandshake:157(4657120ns) {DropHandshakes:469(1456ns) {DropIncomingHandshake:441(424ns) DropOutgoingHandshake:456(303ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:476(3767ns) {SwitchToState:178(1006ns)}}}}}
2025-11-11T17:21:46.203456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519270705085187:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17956591ns) {Configure:104(17950099ns) {ProcessConfigured:135(4800075ns) {StartInitialHandshake:157(4796409ns) {DropHandshakes:469(1835ns) {DropIncomingHandshake:441(427ns) DropOutgoingHandshake:456(326ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:476(5127ns) {SwitchToState:178(2757ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.209548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519276728917137:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(116302444ns) {Configure:104(116296646ns) {ProcessConfigured:135(4329769ns) {StartInitialHandshake:157(4327107ns) {DropHandshakes:469(2669ns) {DropIncomingHandshake:441(501ns) DropOutgoingHandshake:456(497ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:476(2997ns) {SwitchToState:178(764ns)}}}}}
2025-11-11T17:21:46.218142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519276681349919:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21735880ns) {Configure:104(21731422ns) {ProcessConfigured:135(58204ns) {StartInitialHandshake:157(56604ns) {DropHandshakes:469(1849ns) {DropIncomingHandshake:441(351ns) DropOutgoingHandshake:456(224ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:476(1016ns) {SwitchToState:178(548ns)}}}}}
2025-11-11T17:21:46.225572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519277889573376:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(30364478ns) {Configure:104(30360754ns) {ProcessConfigured:135(5997266ns) {StartInitialHandshake:157(5995457ns) {DropHandshakes:469(810ns) {DropIncomingHandshake:441(246ns) DropOutgoingHandshake:456(198ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(2033ns) {SwitchToState:178(700ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.225174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519270705085188:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30222621ns) {Configure:104(30218947ns) {ProcessConfigured:135(7947552ns) {StartInitialHandshake:157(7944330ns) {DropHandshakes:469(2621ns) {DropIncomingHandshake:441(654ns) DropOutgoingHandshake:456(327ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:476(4574ns) {SwitchToState:178(1987ns)}}}}}
2025-11-11T17:21:46.233765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519274020074138:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60016
2025-11-11T17:21:46.240452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519274916417148:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(50126508ns) {Configure:104(50089001ns) {ProcessConfigured:135(5853024ns) {StartInitialHandshake:157(5848535ns) {DropHandshakes:469(1615ns) {DropIncomingHandshake:441(397ns) DropOutgoingHandshake:456(470ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:476(3928ns) {SwitchToState:178(1866ns)}}}}}
2025-11-11T17:21:46.227694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573381:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.241754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519275000052561:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9272
2025-11-11T17:21:46.238082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519273594606017:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(40001787ns) {Configure:104(39996247ns) {ProcessConfigured:135(4116151ns) {StartInitialHandshake:157(4113401ns) {DropHandshakes:469(1043ns) {DropIncomingHandshake:441(212ns) DropOutgoingHandshake:456(217ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:476(2600ns) {SwitchToState:178(824ns)}}}}}
2025-11-11T17:21:46.248110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519269725106774:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(51790467ns) {Configure:104(51782286ns) {ProcessConfigured:135(2539340ns) {StartInitialHandshake:157(2536415ns) {DropHandshakes:469(2542ns) {DropIncomingHandshake:441(747ns) DropOutgoingHandshake:456(382ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:476(4153ns) {SwitchToState:178(1907ns)}}}}}
2025-11-11T17:21:46.243316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519276681349918:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35068688ns) {Configure:104(35064901ns) {ProcessConfigured:135(4306102ns) {StartInitialHandshake:157(4303493ns) {DropHandshakes:469(1707ns) {DropIncomingHandshake:441(379ns) DropOutgoingHandshake:456(330ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:476(3942ns) {SwitchToState:178(1492ns)}}}}}
2025-11-11T17:21:46.254204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519272433949781:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(40172624ns) {Configure:104(40163896ns) {ProcessConfigured:135(5487442ns) {StartInitialHandshake:157(5484947ns) {DropHandshakes:469(971ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(245ns)} GenerateSessionVirtualId:988(389ns) PrepareNewSessionHandshake:476(2472ns) {SwitchToState:178(372ns)}}}}}
2025-11-11T17:21:46.242083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519270621449762:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(52521009ns) {Configure:104(52514974ns) {ProcessConfigured:135(4129221ns) {StartInitialHandshake:157(4125840ns) {DropHandshakes:469(1976ns) {DropIncomingHandshake:441(582ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:476(3049ns) {SwitchToState:178(1151ns)}}}}}
2025-11-11T17:21:46.254125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519270705085186:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(46374069ns) {Configure:104(46367756ns) {ProcessConfigured:135(13557657ns) {StartInitialHandshake:157(13554640ns) {DropHandshakes:469(2017ns) {DropIncomingHandshake:441(413ns) DropOutgoingHandshake:456(377ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:476(1455ns) {SwitchToState:178(79ns)}}}}}
2025-11-11T17:21:46.248853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519274020074135:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45845308ns) {Configure:104(45841408ns) {ProcessConfigured:135(5787229ns) {StartInitialHandshake:157(5784103ns) {DropHandshakes:469(3046ns) {DropIncomingHandshake:441(535ns) DropOutgoingHandshake:456(370ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:476(3902ns) {SwitchToState:178(1443ns)}}}}}
2025-11-11T17:21:46.248075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519272386382558:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(108963078ns) {Configure:104(108944218ns) {ProcessConfigured:135(13845433ns) {StartInitialHandshake:157(13842460ns) {DropHandshakes:469(2638ns) {DropIncomingHandshake:441(682ns) DropOutgoingHandshake:456(368ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:476(4662ns) {SwitchToState:178(2754ns)}}}}}
2025-11-11T17:21:46.254630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519272433949779:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(47024949ns) {Configure:104(47021282ns) {ProcessConfigured:135(4709004ns) {StartInitialHandshake:157(4706251ns) {DropHandshakes:469(1441ns) {DropIncomingHandshake:441(372ns) DropOutgoingHandshake:456(244ns)} GenerateSessionVirtualId:988(2313ns) PrepareNewSessionHandshake:476(3008ns) {SwitchToState:178(1248ns)}}}}}
2025-11-11T17:21:46.247901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519273594606016:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33087516ns) {Configure:104(33081517ns) {ProcessConfigured:135(13020036ns) {StartInitialHandshake:157(13016854ns) {DropHandshakes:469(2939ns) {DropIncomingHandshake:441(1174ns) DropOutgoingHandshake:456(447ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:476(5899ns) {SwitchToState:178(2286ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.255353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519275000052546:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(54875060ns) {Configure:104(54871495ns) {ProcessConfigured:135(15931440ns) {StartInitialHandshake:157(15928579ns) {DropHandshakes:469(3680ns) {DropIncomingHandshake:441(902ns) DropOutgoingHandshake:456(828ns)} GenerateSessionVirtualId:988(844ns) PrepareNewSessionHandshake:476(6718ns) {SwitchToState:178(3234ns)}}}}}
2025-11-11T17:21:46.255440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519274916417147:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53973699ns) {Configure:104(53968426ns) {ProcessConfigured:135(9254633ns) {StartInitialHandshake:157(9251518ns) {DropHandshakes:469(361116ns) {DropIncomingHandshake:441(998ns) DropOutgoingHandshake:456(264ns)} GenerateSessionVirtualId:988(1652ns) PrepareNewSessionHandshake:476(4151ns) {SwitchToState:178(1562ns)}}}}}
2025-11-11T17:21:46.255252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519269725106775:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(61339363ns) {Configure:104(61333406ns) {ProcessConfigured:135(11824853ns) {StartInitialHandshake:157(11822174ns) {DropHandshakes:469(1457ns) {DropIncomingHandshake:441(452ns) DropOutgoingHandshake:456(241ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:476(7855ns) {SwitchToState:178(3337ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.271600Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519275000052551:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.254614Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519277889573381:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.262151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519270621449761:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(56604846ns) {Configure:104(56596951ns) {ProcessConfigured:135(9524718ns) {StartInitialHandshake:157(9521474ns) {DropHandshakes:469(4016ns) {DropIncomingHandshake:441(930ns) DropOutgoingHandshake:456(587ns)} GenerateSessionVirtualId:988(976ns) PrepareNewSessionHandshake:476(2742ns) {SwitchToState:178(818ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.271654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519275000052551:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.270485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519274020074145:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60016
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.256094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519277889573376:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.280685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519274916417161:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34577
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.270684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519274020074147:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60016
2025-11-11T17:21:46.270746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519274020074149:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60016
2025-11-11T17:21:46.280961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519274916417164:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34577
2025-11-11T17:21:46.281932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519274020074133:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(58704359ns) {Configure:104(58699318ns) {ProcessConfigured:135(4538916ns) {StartInitialHandshake:157(4535966ns) {DropHandshakes:469(2055ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(514ns)} GenerateSessionVirtualId:988(649ns) PrepareNewSessionHandshake:476(2940ns) {SwitchToState:178(977ns)}}}}}
2025-11-11T17:21:46.290038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573393:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.282137Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519275000052561:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.283545Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519274020074138:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.298430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917150:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.288213Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417161:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.321372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519276681349934:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53080
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.288193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519270705085186:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.328210Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519276728917150:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.298417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519277889573375:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(74814401ns) {Configure:104(74808630ns) {ProcessConfigured:135(6565780ns) {StartInitialHandshake:157(6563076ns) {DropHandshakes:469(3184ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(412ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:476(5646ns) {SwitchToState:178(3016ns)}}}}}
2025-11-11T17:21:46.330107Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349926:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.291256Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519274020074145:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.291403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519274916417149:2126] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(74655064ns) {Configure:104(74645181ns) {ProcessConfigured:135(15378331ns) {StartInitialHandshake:157(15375768ns) {DropHandshakes:469(1738ns) {DropIncomingHandshake:441(367ns) DropOutgoingHandshake:456(472ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:476(3444ns) {SwitchToState:178(1443ns)}}}}}
2025-11-11T17:21:46.319767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573395:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.309443Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519275000052551:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:46.295706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519274020074149:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.338820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519272433949779:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.321190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519274916417160:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34577
2025-11-11T17:21:46.330151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519276681349926:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.333786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519276681349933:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53080
2025-11-11T17:21:46.302495Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519274020074147:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.355883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519275000052547:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.320341Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519277889573381:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.340618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917155:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.359256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:46.320603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519277889573376:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.333741Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417155:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.352371Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519276728917155:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.310798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519269725106775:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.320681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:46.320853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519277889573376:2108] [node 4] ICP22 created new session: [5:7571519277889573396:2108]
2025-11-11T17:21:46.320911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519277889573396:2108] [node 4] ICS09 handshake done sender: [5:7571519277889573381:2111] self: [5:7571519277889573390:0] peer: [4:7571519275000052550:0] socket: 124 qp: -1
2025-11-11T17:21:46.320949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519277889573396:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:46.321025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:46.333950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573389:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.354909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519277889573389:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.359248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519273594606017:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.364254Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519277889573393:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.377841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519273594606018:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.339135Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349921:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.333792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519274916417155:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.363465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519275000052547:2108] [node 5] ICP22 created new session: [4:7571519275000052566:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.339176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519276681349921:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.365887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519275000052566:2108] [node 5] ICS09 handshake done sender: [4:7571519275000052551:2112] self: [4:7571519275000052550:0] peer: [5:7571519277889573390:0] socket: 122 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.321189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519269725106774:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.382884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573394:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.345545Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417164:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.367422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519275000052566:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:46.338756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519269725106774:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19220337ns) {IncomingHandshake:261(19218170ns) {DropIncomingHandshake:441(253ns) PrepareNewSessionHandshake:476(1035ns) {SwitchToState:178(580ns)} GenerateSessionVirtualId:988(250ns) IssueIncomingHandshakeReply:193(11338696ns) {SwitchToState:178(4935ns)}}}
2025-11-11T17:21:46.355221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519272433949780:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.345790Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519276681349926:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:46.411009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573401:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.357337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917153:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.355228Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519274916417155:2138] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:46.354774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349923:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.369833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519275000052565:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9272
2025-11-11T17:21:46.382835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519274020074150:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.447505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519277889573376:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(126026423ns) {HandleHandshakeStatus:329(126023956ns) {DropIncomingHandshake:441(951ns) DropOutgoingHandshake:456(4717ns) SwitchToState:178(555ns) ProcessPendingSessionEvents:495(120632977ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 120498715ns max# 15575356ns}}}
2025-11-11T17:21:46.355863Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417160:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.363928Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519276728917153:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.382889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519274020074150:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.369862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519275000052564:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9272
2025-11-11T17:21:46.452399Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519277889573394:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.393921Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417163:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.364333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519272433949781:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.354812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519276681349923:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.371029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.386370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519274020074158:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60016
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.364208Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349933:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.371035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.401834Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519275000052561:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:46.393983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519274916417163:2143] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.464012Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519277889573401:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.391949Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519274020074149:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:46.394162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519270621449762:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.442037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917162:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.367690Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519276681349923:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.392268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519269725106774:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.413062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519275000052547:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(56287212ns) {HandleHandshakeStatus:329(56285199ns) {DropIncomingHandshake:441(190ns) DropOutgoingHandshake:456(521ns) SwitchToState:178(142ns) ProcessPendingSessionEvents:495(11832185ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11767372ns max# 11337919ns}}}
2025-11-11T17:21:46.369723Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519276681349921:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:46.468393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573404:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.454566Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519276728917150:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:46.394213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:46.469649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519273594606016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.422166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519275000052564:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.392338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.457100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917158:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.404199Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519274916417163:2143] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:46.373852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519276681349919:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.392400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519269725106774:2048] [node 1] ICP22 created new session: [3:7571519274020074159:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.434567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519270705085186:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.476050Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519277889573389:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:46.404657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519274916417149:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.373929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:46.457236Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519276728917153:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:46.437732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.392459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519274020074159:2048] [node 1] ICS09 handshake done sender: [3:7571519274020074149:2117] self: [3:7571519274020074155:0] peer: [1:7571519276681349920:0] socket: 140 qp: -1
2025-11-11T17:21:46.489934Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.461934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519272433949781:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.404722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:46.392507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519274020074159:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:46.374001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519276681349919:2107] [node 4] ICP22 created new session: [1:7571519276681349939:2107]
2025-11-11T17:21:46.437790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519270705085186:2048] [node 1] ICP22 created new session: [4:7571519275000052570:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.404771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519274916417149:2126] [node 6] ICP22 created new session: [2:7571519274916417173:2126]
2025-11-11T17:21:46.392547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:46.492084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573402:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.465621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:46.374051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519276681349939:2107] [node 4] ICS09 handshake done sender: [1:7571519276681349926:2112] self: [1:7571519276681349925:0] peer: [4:7571519275000052562:0] socket: 127 qp: -1
2025-11-11T17:21:46.452363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519275000052570:2048] [node 1] ICS09 handshake done sender: [4:7571519275000052561:2116] self: [4:7571519275000052562:0] peer: [1:7571519276681349925:0] socket: 128 qp: -1
2025-11-11T17:21:46.404832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519274916417173:2126] [node 6] ICS09 handshake done sender: [2:7571519274916417163:2143] self: [2:7571519274916417162:0] peer: [6:7571519276728917157:0] socket: 148 qp: -1
2025-11-11T17:21:46.507732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519273594606017:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.465689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519272433949781:2050] [node 3] ICP22 created new session: [6:7571519276728917163:2050]
2025-11-11T17:21:46.401679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519274020074137:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.374103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519276681349939:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:46.507898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519273594606018:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519277889573393:2117] held: no
2025-11-11T17:21:46.454157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519275000052570:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:46.374139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:46.401735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519274020074137:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.404868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519274916417173:2126] [node 6] ICS10 traffic start
2025-11-11T17:21:46.465744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519276728917163:2050] [node 3] ICS09 handshake done sender: [6:7571519276728917153:2116] self: [6:7571519276728917159:0] peer: [3:7571519274020074148:0] socket: 144 qp: -1
2025-11-11T17:21:46.454210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:46.404908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:46.519902Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.422923Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519274020074150:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:46.495898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519275000052571:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9272
2025-11-11T17:21:46.465791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519276728917163:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:46.445692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519274020074133:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.377725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349934:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.520322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.507519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519275000052571:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.512558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519270705085187:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.406481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519270621449762:2049] [node 3] ICP22 created new session: [2:7571519274916417170:2049]
2025-11-11T17:21:46.383022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519276681349917:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.465839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:46.450234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:46.523851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519273594606017:2049] [node 2] ICP22 created new session: [5:7571519277889573407:2049]
2025-11-11T17:21:46.466749Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.451633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519274020074157:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60016
2025-11-11T17:21:46.417721Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417151:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.535705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519275000052573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9272
2025-11-11T17:21:46.540181Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519275000052573:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.541816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519270705085188:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.547763Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519275000052559:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.547806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519275000052559:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.548105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519275000052576:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9272
2025-11-11T17:21:46.558693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519275000052579:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9272
2025-11-11T17:21:46.565346Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519275000052559:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:46.565539Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519275000052571:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:46.575651Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.577883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519270705085187:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.577938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:46.578007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519270705085187:2049] [node 2] ICP22 created new session: [4:7571519275000052580:2049]
2025-11-11T17:21:46.578070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519275000052580:2049] [node 2] ICS09 handshake done sender: [4:7571519275000052571:2119] self: [4:7571519275000052575:0] peer: [2:7571519274916417153:0] socket: 177 qp: -1
2025-11-11T17:21:46.578109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519275000052580:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:46.578145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:46.578778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519270705085187:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:46.579268Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519275000052573:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:46.579616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519275000052546:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.579664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:46.579715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519275000052546:2109] [node 6] ICP22 created new session: [4:7571519275000052582:2109]
2025-11-11T17:21:46.579749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519275000052582:2109] [node 6] ICS09 handshake done sender: [4:7571519275000052559:2114] self: [4:7571519275000052557:0] peer: [6:7571519276728917172:0] socket: 145 qp: -1
2025-11-11T17:21:46.579772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519275000052582:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:46.579797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:46.580841Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.582390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519270705085188:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.612685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:46.612735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519270705085188:2050] [node 3] ICP22 created new session: [4:7571519275000052584:2050]
2025-11-11T17:21:46.615062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519275000052584:2050] [node 3] ICS09 handshake done sender: [4:7571519275000052573:2120] self: [4:7571519275000052577:0] peer: [3:7571519274020074139:0] socket: 186 qp: -1
2025-11-11T17:21:46.616242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519275000052584:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:46.616306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:46.403546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519276681349943:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53080
2025-11-11T17:21:46.539033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519277889573407:2049] [node 2] ICS09 handshake done sender: [5:7571519277889573389:2115] self: [5:7571519277889573399:0] peer: [2:7571519274916417150:0] socket: 136 qp: -1
2025-11-11T17:21:46.451956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519274020074133:2109] [node 6] ICP22 created new session: [3:7571519274020074164:2109]
2025-11-11T17:21:46.417784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519274916417151:2137] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.406295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519272386382559:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.539084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519277889573407:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:46.477710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519272433949780:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519276728917155:2117] held: no
2025-11-11T17:21:46.452324Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519274020074137:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:46.471597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519274020074164:2109] [node 6] ICS09 handshake done sender: [3:7571519274020074150:2118] self: [3:7571519274020074148:0] peer: [6:7571519276728917159:0] socket: 143 qp: -1
2025-11-11T17:21:46.471809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519274020074164:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:46.472464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519274020074134:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.472749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:46.473516Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519274020074147:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:46.474105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519274020074134:2108] [node 5] ICP22 created new session: [3:7571519274020074166:2108]
2025-11-11T17:21:46.474134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:46.477710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519274020074166:2108] [node 5] ICS09 handshake done sender: [3:7571519274020074137:2110] self: [3:7571519274020074136:0] peer: [5:7571519277889573400:0] socket: 142 qp: -1
2025-11-11T17:21:46.481886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519274020074166:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:46.491531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519269725106775:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.497719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:46.507653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:46.406402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:46.540519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:46.477752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519276728917158:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.512891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519269725106775:2049] [node 2] ICP22 created new session: [3:7571519274020074169:2049]
2025-11-11T17:21:46.427377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519274916417170:2049] [node 3] ICS09 handshake done sender: [2:7571519274916417155:2138] self: [2:7571519274916417154:0] peer: [3:7571519274020074156:0] socket: 133 qp: -1
2025-11-11T17:21:46.417905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:46.481798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519272433949779:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.523693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519274020074169:2049] [node 2] ICS09 handshake done sender: [3:7571519274020074147:2116] self: [3:7571519274020074156:0] peer: [2:7571519274916417154:0] socket: 137 qp: -1
2025-11-11T17:21:46.434499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519274916417170:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:46.546298Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519277889573392:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.492137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:46.427227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519272386382559:2049] [node 3] ICP22 created new session: [1:7571519276681349945:2049]
2025-11-11T17:21:46.438059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519274916417172:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34577
2025-11-11T17:21:46.678791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519270705085186:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(240848783ns) {HandleHandshakeStatus:329(234396481ns) {DropIncomingHandshake:441(1113ns) DropOutgoingHandshake:456(20511ns) SwitchToState:178(1085ns) ProcessPendingSessionEvents:495(195209734ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 181823383ns max# 9893962ns}}}
2025-11-11T17:21:46.492199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519272433949779:2048] [node 1] ICP22 created new session: [6:7571519276728917165:2048]
2025-11-11T17:21:46.531548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519274020074169:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:46.546341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519277889573392:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.565735Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.427523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519276681349917:2109] [node 6] ICP22 created new session: [1:7571519276681349946:2109]
2025-11-11T17:21:46.531623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:46.548521Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519274020074140:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.548552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519274020074140:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.549690Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519274020074140:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:46.563032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519274020074135:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.565517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:46.569041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519274020074135:2107] [node 4] ICP22 created new session: [3:7571519274020074172:2107]
2025-11-11T17:21:46.572139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519274020074172:2107] [node 4] ICS09 handshake done sender: [3:7571519274020074140:2112] self: [3:7571519274020074139:0] peer: [4:7571519275000052577:0] socket: 184 qp: -1
2025-11-11T17:21:46.573665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519274020074172:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:46.438724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519274916417177:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34577
2025-11-11T17:21:46.576289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:46.622862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519274020074135:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(61706303ns) {HandleHandshakeStatus:329(61701731ns) {DropIncomingHandshake:441(22778ns) DropOutgoingHandshake:456(379ns) SwitchToState:178(557ns) ProcessPendingSessionEvents:495(16552678ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16473469ns max# 7897760ns}}}
2025-11-11T17:21:46.450014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:46.469171Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519274916417151:2137] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:46.472830Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417177:2146] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.473623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519274916417148:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.474048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:46.476186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519274916417148:2107] [node 5] ICP22 created new session: [2:7571519274916417180:2107]
2025-11-11T17:21:46.477777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417172:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.491642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519270621449761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.492196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519274916417180:2107] [node 5] ICS09 handshake done sender: [2:7571519274916417151:2137] self: [2:7571519274916417150:0] peer: [5:7571519277889573399:0] socket: 132 qp: -1
2025-11-11T17:21:46.494251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519274916417180:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:46.499235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:46.503549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519276728917165:2048] [node 1] ICS09 handshake done sender: [6:7571519276728917150:2115] self: [6:7571519276728917156:0] peer: [1:7571519276681349922:0] socket: 141 qp: -1
2025-11-11T17:21:46.534597Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519274916417158:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.534650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519274916417158:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.507396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917161:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.535321Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.508790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519276728917165:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:46.512821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:46.537227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519270621449761:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519274916417172:2145] held: no
2025-11-11T17:21:46.514387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519272433949780:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(40223239ns) {HandleHandshakeStatus:395(40220601ns) {LogHandshakeFail:472(559146ns) {UpdateErrorStateLog:544(5334ns)} DropOutgoingHandshake:456(2013ns) ProcessPendingSessionEvents:495(35907761ns) {unordered DropSessionEvent:512 num# 100 sum# 10508651ns max# 4823112ns}}}
2025-11-11T17:21:46.515146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519276728917137:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.538859Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519274916417158:2139] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:46.515666Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519276728917155:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:46.552429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519274916417147:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.515963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519272433949780:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.554398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:46.516060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:46.554814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519274916417183:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34577
2025-11-11T17:21:46.560120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519274916417147:2110] [node 4] ICP22 created new session: [2:7571519274916417185:2110]
2025-11-11T17:21:46.516113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519272433949780:2049] [node 2] ICP22 created new session: [6:7571519276728917169:2049]
2025-11-11T17:21:46.560191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519274916417185:2110] [node 4] ICS09 handshake done sender: [2:7571519274916417158:2139] self: [2:7571519274916417153:0] peer: [4:7571519275000052575:0] socket: 176 qp: -1
2025-11-11T17:21:46.516158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519276728917169:2049] [node 2] ICS09 handshake done sender: [6:7571519276728917155:2117] self: [6:7571519276728917157:0] peer: [2:7571519274916417162:0] socket: 149 qp: -1
2025-11-11T17:21:46.563254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519274916417185:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:46.516195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519276728917169:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:46.565709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:46.573750Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519274916417172:2145] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:46.516230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:46.587973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519270621449761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.536410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917171:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.588012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:46.582461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519273594606016:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519277889573401:2120] held: no
2025-11-11T17:21:46.590031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519270621449761:2048] [node 1] ICP22 created new session: [2:7571519274916417187:2048]
2025-11-11T17:21:46.436119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519276681349946:2109] [node 6] ICS09 handshake done sender: [1:7571519276681349923:2111] self: [1:7571519276681349922:0] peer: [6:7571519276728917156:0] socket: 125 qp: -1
2025-11-11T17:21:46.606000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519274916417187:2048] [node 1] ICS09 handshake done sender: [2:7571519274916417172:2145] self: [2:7571519274916417181:0] peer: [1:7571519276681349929:0] socket: 174 qp: -1
2025-11-11T17:21:46.606054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519274916417187:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:46.437630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519276681349946:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:46.610900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519270621449762:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(215911608ns) {HandleHandshakeStatus:329(215908454ns) {DropIncomingHandshake:441(306ns) DropOutgoingHandshake:456(580ns) SwitchToState:178(565ns) ProcessPendingSessionEvents:495(143227066ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 143040104ns max# 12085357ns}}}
2025-11-11T17:21:46.539016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519276728917171:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.540448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519276728917139:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:46.562234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519272433949779:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(66033370ns) {HandleHandshakeStatus:329(64603152ns) {DropIncomingHandshake:441(1429ns) DropOutgoingHandshake:456(5269ns) SwitchToState:178(258ns) ProcessPendingSessionEvents:495(5813185ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5748518ns max# 4076319ns}}}
2025-11-11T17:21:46.573655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519272433949781:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(31902552ns) {HandleHandshakeStatus:329(31898356ns) {DropIncomingHandshake:441(717ns) DropOutgoingHandshake:456(8945ns) SwitchToState:178(660ns) ProcessPendingSessionEvents:495(27805166ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27274982ns max# 15650838ns}}}
2025-11-11T17:21:46.690591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519274020074134:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(160935015ns) {HandleHandshakeStatus:329(160931668ns) {DropIncomingHandshake:441(14261ns) DropOutgoingHandshake:456(457ns) SwitchToState:178(409ns) ProcessPendingSessionEvents:495(120911047ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 113306452ns max# 15255823ns}}}
2025-11-11T17:21:46.612218Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14372) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.614756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519276728917139:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14372) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519276728917171:2122] held: no
2025-11-11T17:21:46.630080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917174:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.635388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519276728917173:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57722
2025-11-11T17:21:46.642564Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519276728917171:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:46.655237Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519276728917158:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:46.666178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519276728917139:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.667015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519276728917137:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.668141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:46.668222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519276728917139:2109] [node 4] ICP22 created new session: [6:7571519276728917175:2109]
2025-11-11T17:21:46.668614Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:46.674981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:46.675050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519276728917137:2105] [node 5] ICP22 created new session: [6:7571519276728917176:2105]
2025-11-11T17:21:46.675122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519276728917176:2105] [node 5] ICS09 handshake done sender: [6:7571519276728917158:2118] self: [6:7571519276728917168:0] peer: [5:7571519277889573391:0] socket: 164 qp: -1
2025-11-11T17:21:46.675166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519276728917176:2105] [node 5] ICS10 traffic start
2025-11-11T17:21:46.675228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:46.675984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519276728917137:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:46.615738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519277889573406:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4697
2025-11-11T17:21:46.677452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519276728917175:2109] [node 4] ICS09 handshake done sender: [6:7571519276728917171:2122] self: [6:7571519276728917172:0] peer: [4:7571519275000052557:0] socket: 153 qp: -1
2025-11-11T17:21:46.677901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519276728917175:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:46.679292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:46.681817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519276728917139:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(28479270ns) {HandleHandshakeStatus:329(19068174ns) {DropIncomingHandshake:441(3361ns) DropOutgoingHandshake:456(375ns) SwitchToState:178(829ns) ProcessPendingSessionEvents:495(1407ns)}}
2025-11-11T17:21:46.611630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519270621449762:2049] [node 3] ICP18 handshake (actor [2:7571519274916417177:2146]) is too old
2025-11-11T17:21:46.612232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:46.618373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519270621449761:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(27800733ns) {HandleHandshakeStatus:329(27796725ns) {DropIncomingHandshake:441(590ns) DropOutgoingHandshake:456(305ns) SwitchToState:178(512ns) ProcessPendingSessionEvents:495(7475ns)}}
2025-11-11T17:21:46.437692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:46.625077Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519277889573392:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:46.642308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519273594606018:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(61167319ns) {HandleHandshakeStatus:395(61162911ns) {LogHandshakeFail:472(22690ns) {UpdateErrorStateLog:544(5636ns)} DropOutgoingHandshake:456(1185ns) ProcessPendingSessionEvents:495(55263776ns) {unordered DropSessionEvent:512 num# 100 sum# 55193988ns max# 9295700ns}}}
2025-11-11T17:21:46.642570Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519277889573393:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:46.654320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519277889573375:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.661460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:46.661525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519277889573375:2109] [node 6] ICP22 created new session: [5:7571519277889573410:2109]
2025-11-11T17:21:46.661581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519277889573410:2109] [node 6] ICS09 handshake done sender: [5:7571519277889573392:2116] self: [5:7571519277889573391:0] peer: [6:7571519276728917168:0] socket: 163 qp: -1
2025-11-11T17:21:46.661633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519277889573410:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:46.661676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:46.662703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519273594606018:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.662782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:46.662827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519273594606018:2050] [node 3] ICP22 created new session: [5:7571519277889573412:2050]
2025-11-11T17:21:46.662881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519277889573412:2050] [node 3] ICS09 handshake done sender: [5:7571519277889573393:2117] self: [5:7571519277889573400:0] peer: [3:7571519274020074136:0] socket: 146 qp: -1
2025-11-11T17:21:46.662913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519277889573412:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:46.662950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:46.669685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519273594606017:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(160068070ns) {HandleHandshakeStatus:329(160063948ns) {DropIncomingHandshake:441(629ns) DropOutgoingHandshake:456(5168ns) SwitchToState:178(528ns) ProcessPendingSessionEvents:495(121382125ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 121237283ns max# 22741346ns}}}
2025-11-11T17:21:46.669761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519273594606017:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:46.709540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519273594606016:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(66830603ns) {HandleHandshakeStatus:395(66826903ns) {LogHandshakeFail:472(387392ns) {UpdateErrorStateLog:544(5378ns)} DropOutgoingHandshake:456(1230ns) ProcessPendingSessionEvents:495(60286857ns) {unordered DropSessionEvent:512 num# 100 sum# 14563810ns max# 7957030ns}}}
2025-11-11T17:21:46.709690Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519277889573401:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:46.714527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519273594606016:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:46.724161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:46.724223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519273594606016:2048] [node 1] ICP22 created new session: [5:7571519277889573414:2048]
2025-11-11T17:21:46.724294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519277889573414:2048] [node 1] ICS09 handshake done sender: [5:7571519277889573401:2120] self: [5:7571519277889573405:0] peer: [1:7571519276681349927:0] socket: 171 qp: -1
2025-11-11T17:21:46.724335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519277889573414:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:46.724383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:46.439416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519276681349945:2049] [node 3] ICS09 handshake done sender: [1:7571519276681349921:2110] self: [1:7571519276681349920:0] peer: [3:7571519274020074155:0] socket: 138 qp: -1
2025-11-11T17:21:46.439464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519276681349945:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:46.451684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349943:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.454652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:46.462226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519276681349919:2107] [node 4] ICP18 handshake (actor [1:7571519276681349943:2117]) is too old
2025-11-11T17:21:46.474652Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349928:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.474670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519276681349928:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.475155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519276681349948:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53080
2025-11-11T17:21:46.475448Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519276681349928:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:46.499342Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349948:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.502235Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519276681349931:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:46.502265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519276681349931:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:46.503393Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519276681349931:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:46.503936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519276681349918:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.507835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:46.511885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519276681349918:2108] [node 5] ICP22 created new session: [1:7571519276681349953:2108]
2025-11-11T17:21:46.513142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519276681349953:2108] [node 5] ICS09 handshake done sender: [1:7571519276681349928:2113] self: [1:7571519276681349927:0] peer: [5:7571519277889573405:0] socket: 170 qp: -1
2025-11-11T17:21:46.519870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519276681349953:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:46.526778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519272386382558:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:46.526882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:46.526934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519272386382558:2048] [node 2] ICP22 created new session: [1:7571519276681349955:2048]
2025-11-11T17:21:46.526985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519276681349955:2048] [node 2] ICS09 handshake done sender: [1:7571519276681349931:2114] self: [1:7571519276681349929:0] peer: [2:7571519274916417181:0] socket: 173 qp: -1
2025-11-11T17:21:46.527022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519276681349955:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:46.527080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:46.529337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:46.586597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519276681349918:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(86343877ns) {HandleHandshakeStatus:329(86339990ns) {DropIncomingHandshake:441(18866ns) DropOutgoingHandshake:456(3012ns) SwitchToState:178(792ns) ProcessPendingSessionEvents:495(41410245ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39255259ns max# 10884417ns}}}
2025-11-11T17:21:46.708150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519276681349917:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(268645260ns) {HandleHandshakeStatus:329(268641570ns) {DropIncomingHandshake:441(1645ns) DropOutgoingHandshake:456(1888ns) SwitchToState:178(1031ns) ProcessPendingSessionEvents:495(189296045ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 186718552ns max# 18583084ns}}}
2025-11-11T17:21:46.708858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519272386382558:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(179413684ns) {HandleHandshakeStatus:329(179402692ns) {DropIncomingHandshake:441(15740ns) DropOutgoingHandshake:456(504ns) SwitchToState:178(4459ns) ProcessPendingSessionEvents:495(178922596ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 171718824ns max# 12609201ns}}}
2025-11-11T17:21:46.722563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519272386382559:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(243393378ns) {HandleHandshakeStatus:329(243388276ns) {DropIncomingHandshake:441(32737ns) DropOutgoingHandshake:456(579ns) SwitchToState:178(629ns) ProcessPendingSessionEvents:495(173513620ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 173327283ns max# 15534099ns}}}
2025-11-11T17:21:46.727921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519269725106775:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(167895521ns) {HandleHandshakeStatus:329(167863163ns) {DropIncomingHandshake:441(2330ns) DropOutgoingHandshake:456(10851ns) SwitchToState:178(752ns) ProcessPendingSessionEvents:495(102885248ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 102726641ns max# 15231423ns}}}
2025-11-11T17:21:46.728710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519274020074133:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(234091693ns) {HandleHandshakeStatus:329(234089345ns) {DropIncomingHandshake:441(495ns) DropOutgoingHandshake:456(629ns) SwitchToState:178(3358ns) ProcessPendingSessionEvents:495(181989058ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 181797724ns max# 11557053ns}}}
2025-11-11T17:21:46.725927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519274916417148:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(228174832ns) {HandleHandshakeStatus:329(228172593ns) {DropIncomingHandshake:441(15084ns) DropOutgoingHandshake:456(680ns) SwitchToState:178(221ns) ProcessPendingSessionEvents:495(190099995ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 189888658ns max# 15222315ns}}}
2025-11-11T17:21:46.729385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519270705085188:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(136785019ns) {HandleHandshakeStatus:329(135525582ns) {DropIncomingHandshake:441(318ns) DropOutgoingHandshake:456(5609018ns) SwitchToState:178(1113ns) ProcessPendingSessionEvents:495(82565320ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 72296979ns max# 16622585ns}}}
2025-11-11T17:21:46.729493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519270705085188:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:46.730154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519274916417147:2110] [node 4] ICP35 event processing took too much time PendingConnection:278(176771194ns) {HandleHandshakeStatus:329(176768421ns) {DropIncomingHandshake:441(12416ns) DropOutgoingHandshake:456(311ns) SwitchToState:178(344ns) ProcessPendingSessionEvents:495(157188742ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 156995571ns max# 13959079ns}}}
2025-11-11T17:21:46.815483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:46.815711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519274916417187:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-11-11T17:21:46.815787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519270621449761:2048] [node 1] ICP30 unregister session Session# [2:7571519274916417187:2048] VirtualId# [2:7571519274916417181:0]
2025-11-11T17:21:46.815824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519274916417187:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.815930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:46.815968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:46.817539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:46.817812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519274020074159:2048] [node 1] ICS01 socket: 140 reason# EndOfStream
2025-11-11T17:21:46.817846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519269725106774:2048] [node 1] ICP30 unregister session Session# [3:7571519274020074159:2048] VirtualId# [3:7571519274020074155:0]
2025-11-11T17:21:46.817881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519274020074159:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.817954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:46.817989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:46.825773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:46.826004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519277889573414:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-11T17:21:46.826085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519273594606016:2048] [node 1] ICP30 unregister session Session# [5:7571519277889573414:2048] VirtualId# [5:7571519277889573405:0]
2025-11-11T17:21:46.826144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519277889573414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.826226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:46.826277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:46.825782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:46.826002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519275000052570:2048] [node 1] ICS07 socket disconnect 128 reason# EndOfStream
2025-11-11T17:21:46.826032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519275000052570:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.826140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:46.826184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519275000052570:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:46.826960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:46.827178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519276728917165:2048] [node 1] ICS01 socket: 141 reason# EndOfStream
2025-11-11T17:21:46.827212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519272433949779:2048] [node 1] ICP30 unregister session Session# [6:7571519276728917165:2048] VirtualId# [6:7571519276728917156:0]
2025-11-11T17:21:46.827240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519276728917165:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.827299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:46.827331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:46.829911Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Couldn't connect to any resolved address
2025-11-11T17:21:46.830047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519270705085186:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:46.833245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519275000052570:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:46.833728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519270705085186:2048] [node 1] ICP30 unregister session Session# [4:7571519275000052570:2048] VirtualId# [4:7571519275000052562:0]
2025-11-11T17:21:46.833930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:46.833993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519270705085186:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:46.834024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:46.844274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519270705085186:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:46.880421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:46.880688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519274020074169:2049] [node 2] ICS07 socket disconnect 137 reason# EndOfStream
2025-11-11T17:21:46.880718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519274020074169:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.880808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:46.880847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519274020074169:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:46.885482Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:21:46.888122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:46.888514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519275000052580:2049] [node 2] ICS01 socket: 177 reason# EndOfStream
2025-11-11T17:21:46.885619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519269725106775:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:46.888882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:46.889404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519277889573407:2049] [node 2] ICS01 socket: 136 reason# EndOfStream
2025-11-11T17:21:46.889490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519273594606017:2049] [node 2] ICP30 unregister session Session# [5:7571519277889573407:2049] VirtualId# [5:7571519277889573399:0]
2025-11-11T17:21:46.885675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519274020074169:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:46.889523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519277889573407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.888565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519270705085187:2049] [node 2] ICP30 unregister session Session# [4:7571519275000052580:2049] VirtualId# [4:7571519275000052575:0]
2025-11-11T17:21:46.889627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:46.888606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519275000052580:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.889684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:46.888684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:46.888730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:46.890793Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Couldn't connect to any resolved address
2025-11-11T17:21:46.890898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519269725106775:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:46.891005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519274020074169:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:46.891091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519269725106775:2049] [node 2] ICP30 unregister session Session# [3:7571519274020074169:2049] VirtualId# [3:7571519274020074156:0]
2025-11-11T17:21:46.891232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:46.891276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519269725106775:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:46.891289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4487) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:46.891001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:46.891214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519276728917169:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-11T17:21:46.891248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519272433949780:2049] [node 2] ICP30 unregister session Session# [6:7571519276728917169:2049] VirtualId# [6:7571519276728917157:0]
2025-11-11T17:21:46.891277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519276728917169:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.891361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:46.891401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:46.900508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519269725106775:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:46.980889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:46.981141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519277889573412:2050] [node 3] ICS01 socket: 146 reason# EndOfStream
2025-11-11T17:21:46.981227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519273594606018:2050] [node 3] ICP30 unregister session Session# [5:7571519277889573412:2050] VirtualId# [5:7571519277889573400:0]
2025-11-11T17:21:46.981274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519277889573412:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.981363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:46.981408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:46.981679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:46.980883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:46.981128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519275000052584:2050] [node 3] ICS07 socket disconnect 186 reason# EndOfStream
2025-11-11T17:21:46.981157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519275000052584:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.981248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:46.981292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519275000052584:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:46.981861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519276728917163:2050] [node 3] ICS01 socket: 144 reason# EndOfStream
2025-11-11T17:21:46.981907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519272433949781:2050] [node 3] ICP30 unregister session Session# [6:7571519276728917163:2050] VirtualId# [6:7571519276728917159:0]
2025-11-11T17:21:46.981929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519276728917163:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:46.981985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:46.982026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:46.982567Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28906) Couldn't connect to any resolved address
2025-11-11T17:21:46.982817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519270705085188:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28906) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:46.982864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519275000052584:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:46.982896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519270705085188:2050] [node 3] ICP30 unregister session Session# [4:7571519275000052584:2050] VirtualId# [4:7571519275000052577:0]
2025-11-11T17:21:46.982986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:46.983016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519270705085188:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28906) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:46.983051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28906) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:47.045768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:47.045963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519277889573396:2108] [node 4] ICS07 socket disconnect 124 reason# EndOfStream
2025-11-11T17:21:47.045995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519277889573396:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:47.046085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:47.046136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519277889573396:2108] [node 4] ICS15 start handshake
2025-11-11T17:21:47.049825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:47.049833Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14372) Couldn't connect to any resolved address
2025-11-11T17:21:47.050069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519276728917175:2109] [node 4] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:21:47.050139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519277889573376:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14372) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:47.050125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519276728917139:2109] [node 4] ICP30 unregister session Session# [6:7571519276728917175:2109] VirtualId# [6:7571519276728917172:0]
2025-11-11T17:21:47.050192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519277889573396:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:47.050162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519276728917175:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:47.050223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519277889573376:2108] [node 4] ICP30 unregister session Session# [5:7571519277889573396:2108] VirtualId# [5:7571519277889573390:0]
2025-11-11T17:21:47.050253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:47.050257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:47.050278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519277889573376:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14372) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:47.050295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:47.050288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14372) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:47.061693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519277889573376:2108] [node 4] ICP33 wake up from error state
2025-11-11T17:21:47.138900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:47.139141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519276728917176:2105] [node 5] ICS07 socket disconnect 164 reason# EndOfStream
2025-11-11T17:21:47.139170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519276728917176:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:47.139251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:47.139296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519276728917176:2105] [node 5] ICS15 start handshake
2025-11-11T17:21:47.143303Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22802) Couldn't connect to any resolved address
2025-11-11T17:21:47.144452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519276728917137:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22802) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:47.144513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519276728917176:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:47.144556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519276728917137:2105] [node 5] ICP30 unregister session Session# [6:7571519276728917176:2105] VirtualId# [6:7571519276728917168:0]
2025-11-11T17:21:47.144600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:21:47.144633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519276728917137:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:47.144647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:47.157190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519276728917137:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:47.227635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519278790905918:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:47.227679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519278790905919:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:47.249852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519278328374299:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:47.249899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519278328374300:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:47.271218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519282079780201:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:47.271274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519282079780202:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:47.292413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519280779551612:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:47.292460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519280779551613:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:47.292488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519280779551614:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:47.314123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519281632695144:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:47.314176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519281632695145:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:47.314216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519281632695146:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:47.340199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519279345359618:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:47.340256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519279345359619:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:47.340349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519279345359620:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:47.511199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519279345359669:2091] [node 4] ICP01 ready to work
2025-11-11T17:21:47.529837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519278790905981:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:47.536938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519278328374362:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:47.537430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519278790905982:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:47.537712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519282079780271:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:47.539742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519280779551675:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:47.539958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519282079780272:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:47.540441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519279345359678:2093] [node 5] ICP01 ready to work
2025-11-11T17:21:47.544009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519281632695208:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:47.541346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519280779551676:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:47.540785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519282079780273:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:47.545883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519278790905983:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:47.548995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519281632695209:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:47.557658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519278328374365:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:47.562083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519279345359620:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21174843ns) {Configure:104(21167737ns) {ProcessConfigured:135(57021ns) {StartInitialHandshake:157(53848ns) {DropHandshakes:469(1452ns) {DropIncomingHandshake:441(361ns) DropOutgoingHandshake:456(240ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:476(1635ns) {SwitchToState:178(574ns)}}}}}
2025-11-11T17:21:47.561427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519278328374366:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:47.586466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519278790905918:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30133221ns) {Configure:104(30128333ns) {ProcessConfigured:135(20099526ns) {StartInitialHandshake:157(20096495ns) {DropHandshakes:469(3379ns) {DropIncomingHandshake:441(432ns) DropOutgoingHandshake:456(615ns)} GenerateSessionVirtualId:988(1782ns) PrepareNewSessionHandshake:476(5356ns) {SwitchToState:178(2294ns)}}}}}
2025-11-11T17:21:47.590782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519278328374362:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18426903ns) {Configure:104(18423259ns) {ProcessConfigured:135(54126ns) {StartInitialHandshake:157(51728ns) {DropHandshakes:469(1163ns) {DropIncomingHandshake:441(230ns) DropOutgoingHandshake:456(161ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:476(834ns) {SwitchToState:178(263ns)}}}}}
2025-11-11T17:21:47.595782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519281632695144:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33369247ns) {Configure:104(33365406ns) {ProcessConfigured:135(865089ns) {StartInitialHandshake:157(862823ns) {DropHandshakes:469(1063ns) {DropIncomingHandshake:441(301ns) DropOutgoingHandshake:456(197ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:476(2170ns) {SwitchToState:178(1081ns)}}}}}
2025-11-11T17:21:47.595563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519282079780271:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37034215ns) {Configure:104(37029269ns) {ProcessConfigured:135(471210ns) {StartInitialHandshake:157(467746ns) {DropHandshakes:469(1471ns) {DropIncomingHandshake:441(259ns) DropOutgoingHandshake:456(176ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:476(3189ns) {SwitchToState:178(1128ns)}}}}}
2025-11-11T17:21:47.596404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519279345359669:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16469143ns) {Configure:104(16462976ns) {ProcessConfigured:135(2795590ns) {StartInitialHandshake:157(2792686ns) {DropHandshakes:469(1724ns) {DropIncomingHandshake:441(742ns) DropOutgoingHandshake:456(309ns)} GenerateSessionVirtualId:988(595ns) PrepareNewSessionHandshake:476(2619ns) {SwitchToState:178(1314ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.601937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519282079780284:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30731
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.607424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519278790905994:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22897
2025-11-11T17:21:47.618816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519278790905995:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22897
2025-11-11T17:21:47.627347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519279345359678:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30358433ns) {Configure:104(30354305ns) {ProcessConfigured:135(58842ns) {StartInitialHandshake:157(56353ns) {DropHandshakes:469(2602ns) {DropIncomingHandshake:441(774ns) DropOutgoingHandshake:456(366ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:476(1835ns) {SwitchToState:178(537ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.636582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519282079780288:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30731
2025-11-11T17:21:47.636657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519282079780289:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30731
2025-11-11T17:21:47.650040Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905995:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.654161Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519282079780284:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.654338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519282079780202:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.654105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519279345359619:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25873025ns) {Configure:104(25865415ns) {ProcessConfigured:135(6706585ns) {StartInitialHandshake:157(6696294ns) {DropHandshakes:469(3885ns) {DropIncomingHandshake:441(821ns) DropOutgoingHandshake:456(646ns)} GenerateSessionVirtualId:988(1016ns) PrepareNewSessionHandshake:476(3725ns) {SwitchToState:178(1578ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.666973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519278790905999:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22897
2025-11-11T17:21:47.677966Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905994:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.668632Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374368:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.668691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519278328374368:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.685922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695223:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.679607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519278328374378:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.698040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551687:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.679957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519278328374379:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-11T17:21:47.702420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695224:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.698198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551688:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
2025-11-11T17:21:47.704908Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.706371Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519282079780289:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.680007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519278328374380:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-11T17:21:47.702601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695225:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.706727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519282079780202:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519282079780284:2122] held: no
2025-11-11T17:21:47.680156Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374378:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.698918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551691:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
2025-11-11T17:21:47.707958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519282079780291:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30731
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.704746Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519281632695224:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.712162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519278790906000:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22897
2025-11-11T17:21:47.698976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551692:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
2025-11-11T17:21:47.680635Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519278328374368:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:47.713525Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790906000:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.699150Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519280779551687:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.707618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519281632695145:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.687229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519278328374300:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.729767Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905989:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.729824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519278790905989:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.738375Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905999:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.738915Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519278790905989:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:47.713005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519280779551613:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.713139Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519280779551688:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.713570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519280779551614:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.713757Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519280779551691:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.745852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519282079780292:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30731
2025-11-11T17:21:47.713843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519280779551612:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.717739Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519280779551692:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.746833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551698:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
2025-11-11T17:21:47.746787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695227:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.687324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:47.687382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519278328374300:2049] [node 3] ICP22 created new session: [2:7571519278328374383:2049]
2025-11-11T17:21:47.687424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519278328374383:2049] [node 3] ICS09 handshake done sender: [2:7571519278328374368:2111] self: [2:7571519278328374367:0] peer: [3:7571519282079780290:0] socket: 127 qp: -1
2025-11-11T17:21:47.687452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519278328374383:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:47.687500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:47.698161Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374380:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.701419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519278328374387:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-11T17:21:47.708324Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374372:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.748246Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519281632695223:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.748361Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519281632695225:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.748454Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519281632695224:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:47.708365Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519278328374372:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.709569Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519278328374372:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:47.762424Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519280779551691:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:47.747451Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374379:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.762650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519278790905981:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.762735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:47.762791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519278790905981:2107] [node 4] ICP22 created new session: [1:7571519278790906006:2107]
2025-11-11T17:21:47.752293Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374370:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.763161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519278790906006:2107] [node 4] ICS09 handshake done sender: [1:7571519278790905989:2112] self: [1:7571519278790905988:0] peer: [4:7571519280779551696:0] socket: 140 qp: -1
2025-11-11T17:21:47.763233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519278790906006:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:47.763496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:47.763203Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519282079780288:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.764062Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519282079780279:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.763720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519280779551612:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.752330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519278328374370:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.764089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519282079780279:2119] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.759991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519278328374365:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.763778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:47.764870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359696:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.760126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:47.760194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519278328374365:2108] [node 5] ICP22 created new session: [2:7571519278328374389:2108]
2025-11-11T17:21:47.760247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519278328374389:2108] [node 5] ICS09 handshake done sender: [2:7571519278328374372:2113] self: [2:7571519278328374371:0] peer: [5:7571519281632695226:0] socket: 133 qp: -1
2025-11-11T17:21:47.764038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519280779551612:2048] [node 1] ICP22 created new session: [4:7571519280779551699:2048]
2025-11-11T17:21:47.762972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519281632695145:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.764089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519280779551699:2048] [node 1] ICS09 handshake done sender: [4:7571519280779551691:2117] self: [4:7571519280779551696:0] peer: [1:7571519278790905988:0] socket: 149 qp: -1
2025-11-11T17:21:47.763033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:47.764119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519280779551699:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:47.770751Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519282079780292:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.770971Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519282079780284:2122] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:47.771665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519282079780202:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.771755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:47.771809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519282079780202:2049] [node 2] ICP22 created new session: [3:7571519282079780296:2049]
2025-11-11T17:21:47.771861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519282079780296:2049] [node 2] ICS09 handshake done sender: [3:7571519282079780284:2122] self: [3:7571519282079780290:0] peer: [2:7571519278328374367:0] socket: 129 qp: -1
2025-11-11T17:21:47.771895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519282079780296:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:47.771926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:47.772650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519282079780201:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.773827Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519282079780279:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:47.760291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519278328374389:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:47.763184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519281632695145:2049] [node 2] ICP22 created new session: [5:7571519281632695228:2049]
2025-11-11T17:21:47.764147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:47.777925Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905985:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.781024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519282079780271:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.760343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:47.763441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519281632695228:2049] [node 2] ICS09 handshake done sender: [5:7571519281632695224:2117] self: [5:7571519281632695226:0] peer: [2:7571519278328374371:0] socket: 158 qp: -1
2025-11-11T17:21:47.761745Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374387:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.782476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:47.763478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519281632695228:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:47.777962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519278790905985:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.782743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519282079780271:2107] [node 4] ICP22 created new session: [3:7571519282079780300:2107]
2025-11-11T17:21:47.763981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519278328374299:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.787733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519282079780300:2107] [node 4] ICS09 handshake done sender: [3:7571519282079780279:2119] self: [3:7571519282079780278:0] peer: [4:7571519280779551695:0] socket: 132 qp: -1
2025-11-11T17:21:47.787790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519282079780300:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:47.779431Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905987:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.764912Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519278328374370:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:47.792257Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519279345359696:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.792978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519279345359619:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.763670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:47.779452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519278790905987:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.773747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519278328374362:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.773830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:47.773875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519278328374362:2107] [node 4] ICP22 created new session: [2:7571519278328374395:2107]
2025-11-11T17:21:47.773920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519278328374395:2107] [node 4] ICS09 handshake done sender: [2:7571519278328374370:2112] self: [2:7571519278328374369:0] peer: [4:7571519280779551694:0] socket: 128 qp: -1
2025-11-11T17:21:47.773958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519278328374395:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:47.773991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:47.784475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519278328374397:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-11T17:21:47.791891Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519278328374379:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:47.781978Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519278790905985:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:47.788384Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519278790905987:2111] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:47.788662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519278790905918:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.788751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:47.788798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519278790905918:2048] [node 2] ICP22 created new session: [1:7571519278790906011:2048]
2025-11-11T17:21:47.788843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519278790906011:2048] [node 2] ICS09 handshake done sender: [1:7571519278790905985:2110] self: [1:7571519278790905984:0] peer: [2:7571519278328374393:0] socket: 138 qp: -1
2025-11-11T17:21:47.788873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519278790906011:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:47.788907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:47.790017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519278790905919:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.790104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:47.790144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519278790905919:2049] [node 3] ICP22 created new session: [1:7571519278790906013:2049]
2025-11-11T17:21:47.790189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519278790906013:2049] [node 3] ICS09 handshake done sender: [1:7571519278790905987:2111] self: [1:7571519278790905986:0] peer: [3:7571519282079780298:0] socket: 139 qp: -1
2025-11-11T17:21:47.790213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519278790906013:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:47.790241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:47.792235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:47.774339Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.803489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359698:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.799000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519282079780271:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(17122802ns) {HandleHandshakeStatus:329(17119328ns) {DropIncomingHandshake:441(654ns) DropOutgoingHandshake:456(1008ns) SwitchToState:178(986ns) ProcessPendingSessionEvents:495(1710054ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1647073ns max# 1199175ns}}}
2025-11-11T17:21:47.800703Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.809203Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.816903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519282079780201:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519282079780288:2123] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.818098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519282079780302:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30731
2025-11-11T17:21:47.784738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519281632695208:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.818909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519279345359698:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.819565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519281632695145:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:47.819230Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519282079780288:2123] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.821722Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.820789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695232:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.820929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695233:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.822969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519279345359618:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.823061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359699:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.824007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519281632695233:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.825196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519281632695144:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.826547Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519280779551680:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.826580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519280779551680:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.827394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519280779551613:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519280779551687:2115] held: no
2025-11-11T17:21:47.823635Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905993:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.828389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551701:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.829423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
2025-11-11T17:21:47.834856Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519279345359699:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.835051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519279345359620:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.823664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519278790905993:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.829938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519282079780201:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.830033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:47.830107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519282079780201:2048] [node 1] ICP22 created new session: [3:7571519282079780304:2048]
2025-11-11T17:21:47.830158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519282079780304:2048] [node 1] ICS09 handshake done sender: [3:7571519282079780288:2123] self: [3:7571519282079780298:0] peer: [1:7571519278790905986:0] socket: 145 qp: -1
2025-11-11T17:21:47.830194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519282079780304:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:47.830233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:47.841353Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519282079780283:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.841386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519282079780283:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.842052Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519282079780281:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.842074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519282079780281:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.835218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519278328374299:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.833641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519281632695232:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.848558Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519278790905991:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.848593Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519278790905991:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.833823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519281632695146:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.851096Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519280779551687:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:47.851440Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519278790905993:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:47.841983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.851929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359702:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.852010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.853397Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519282079780283:2121] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:47.842057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519278328374299:2048] [node 1] ICP22 created new session: [2:7571519278328374398:2048]
2025-11-11T17:21:47.842110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519278328374398:2048] [node 1] ICS09 handshake done sender: [2:7571519278328374379:2116] self: [2:7571519278328374393:0] peer: [1:7571519278790905984:0] socket: 147 qp: -1
2025-11-11T17:21:47.845716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519278328374398:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:47.850932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:47.855871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519278328374299:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19871472ns) {HandleHandshakeStatus:329(19868849ns) {DropIncomingHandshake:441(388ns) DropOutgoingHandshake:456(6616ns) SwitchToState:178(206ns) ProcessPendingSessionEvents:495(3633912ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3571320ns max# 2052295ns}}}
2025-11-11T17:21:47.856029Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519278328374374:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.856052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519278328374374:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.857118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519278328374299:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:47.858022Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519280779551688:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:47.859077Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519282079780281:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:47.858285Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519278790905991:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:47.859952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519278790905983:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.864796Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519279345359698:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:47.859991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.860020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519278790905983:2109] [node 6] ICP22 created new session: [1:7571519278790906018:2109]
2025-11-11T17:21:47.868074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519280779551703:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51561
2025-11-11T17:21:47.868339Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519280779551680:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:47.868805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519280779551613:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.860079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519278790906018:2109] [node 6] ICS09 handshake done sender: [1:7571519278790905993:2114] self: [1:7571519278790905992:0] peer: [6:7571519279345359700:0] socket: 134 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.860132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519278790906018:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.869140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519282079780308:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30731
2025-11-11T17:21:47.868995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:47.869894Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519278328374374:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:47.869814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519280779551613:2049] [node 2] ICP22 created new session: [4:7571519280779551707:2049]
2025-11-11T17:21:47.860159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:47.860727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519278790905982:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.860759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:47.860783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519278790905982:2108] [node 5] ICP22 created new session: [1:7571519278790906020:2108]
2025-11-11T17:21:47.860809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519278790906020:2108] [node 5] ICS09 handshake done sender: [1:7571519278790905991:2113] self: [1:7571519278790905990:0] peer: [5:7571519281632695235:0] socket: 162 qp: -1
2025-11-11T17:21:47.860829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519278790906020:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:47.860846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:47.870622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695237:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.870155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519279345359618:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.870247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:47.870291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519279345359618:2048] [node 1] ICP22 created new session: [6:7571519279345359705:2048]
2025-11-11T17:21:47.870338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519279345359705:2048] [node 1] ICS09 handshake done sender: [6:7571519279345359698:2116] self: [6:7571519279345359700:0] peer: [1:7571519278790905992:0] socket: 159 qp: -1
2025-11-11T17:21:47.870367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519279345359705:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:47.870403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:47.869197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519282079780273:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.869278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:47.869307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519282079780273:2109] [node 6] ICP22 created new session: [3:7571519282079780309:2109]
2025-11-11T17:21:47.869348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519282079780309:2109] [node 6] ICS09 handshake done sender: [3:7571519282079780283:2121] self: [3:7571519282079780282:0] peer: [6:7571519279345359701:0] socket: 130 qp: -1
2025-11-11T17:21:47.869371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519282079780309:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:47.869401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:47.871259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519280779551707:2049] [node 2] ICS09 handshake done sender: [4:7571519280779551687:2115] self: [4:7571519280779551694:0] peer: [2:7571519278328374369:0] socket: 136 qp: -1
2025-11-11T17:21:47.871296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519280779551707:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:47.871331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:47.871961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519280779551614:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.872025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.872431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695238:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.872502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359704:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.873565Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519279345359699:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:47.874001Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.872258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519282079780272:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.872810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519280779551614:2050] [node 3] ICP22 created new session: [4:7571519280779551709:2050]
2025-11-11T17:21:47.872858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519280779551709:2050] [node 3] ICS09 handshake done sender: [4:7571519280779551688:2116] self: [4:7571519280779551695:0] peer: [3:7571519282079780278:0] socket: 137 qp: -1
2025-11-11T17:21:47.872889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519280779551709:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:47.875559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519281632695144:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519281632695233:2121] held: no
2025-11-11T17:21:47.873801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:47.875036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.876609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519278328374366:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.876689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:47.876724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519278328374366:2109] [node 6] ICP22 created new session: [2:7571519278328374401:2109]
2025-11-11T17:21:47.876766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519278328374401:2109] [node 6] ICS09 handshake done sender: [2:7571519278328374374:2114] self: [2:7571519278328374373:0] peer: [6:7571519279345359697:0] socket: 168 qp: -1
2025-11-11T17:21:47.876794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519278328374401:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:47.875088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519282079780272:2108] [node 5] ICP22 created new session: [3:7571519282079780311:2108]
2025-11-11T17:21:47.876342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519282079780311:2108] [node 5] ICS09 handshake done sender: [3:7571519282079780281:2120] self: [3:7571519282079780280:0] peer: [5:7571519281632695236:0] socket: 170 qp: -1
2025-11-11T17:21:47.876669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519282079780311:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:47.876703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:47.876556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359707:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.879117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519280779551675:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.879165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:47.879201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519280779551675:2108] [node 5] ICP22 created new session: [4:7571519280779551711:2108]
2025-11-11T17:21:47.879245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519280779551711:2108] [node 5] ICS09 handshake done sender: [4:7571519280779551680:2111] self: [4:7571519280779551679:0] peer: [5:7571519281632695231:0] socket: 160 qp: -1
2025-11-11T17:21:47.879272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519280779551711:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:47.879300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:47.877154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519281632695239:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-11T17:21:47.879983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:47.884178Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519281632695225:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:47.884452Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519281632695233:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:47.884931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519279345359620:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.885015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:47.897483Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2936) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.898488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519281632695208:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.898570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:47.898619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519281632695208:2108] [node 4] ICP22 created new session: [5:7571519281632695240:2108]
2025-11-11T17:21:47.898668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519281632695240:2108] [node 4] ICS09 handshake done sender: [5:7571519281632695225:2118] self: [5:7571519281632695231:0] peer: [4:7571519280779551679:0] socket: 163 qp: -1
2025-11-11T17:21:47.898705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519281632695240:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:47.898744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:47.900182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519281632695144:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.900261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:47.900307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519281632695144:2048] [node 1] ICP22 created new session: [5:7571519281632695242:2048]
2025-11-11T17:21:47.900350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519281632695242:2048] [node 1] ICS09 handshake done sender: [5:7571519281632695233:2121] self: [5:7571519281632695235:0] peer: [1:7571519278790905990:0] socket: 176 qp: -1
2025-11-11T17:21:47.900380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519281632695242:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:47.900410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:47.901071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519281632695232:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:47.901383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519281632695146:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2936) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519281632695232:2120] held: no
2025-11-11T17:21:47.891512Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519280779551703:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.885065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519279345359620:2050] [node 3] ICP22 created new session: [6:7571519279345359708:2050]
2025-11-11T17:21:47.902904Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.905704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519278328374403:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-11T17:21:47.885226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519279345359708:2050] [node 3] ICS09 handshake done sender: [6:7571519279345359699:2117] self: [6:7571519279345359701:0] peer: [3:7571519282079780282:0] socket: 177 qp: -1
2025-11-11T17:21:47.903590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519281632695146:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.903673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:47.903713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519281632695146:2050] [node 3] ICP22 created new session: [5:7571519281632695245:2050]
2025-11-11T17:21:47.903750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519281632695245:2050] [node 3] ICS09 handshake done sender: [5:7571519281632695232:2120] self: [5:7571519281632695236:0] peer: [3:7571519282079780280:0] socket: 175 qp: -1
2025-11-11T17:21:47.903791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519281632695245:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:47.903817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:47.885564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519279345359708:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:47.885616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:47.887095Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2936) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.902500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519279345359620:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:47.902750Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519279345359696:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:47.908621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519279345359619:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.908718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:47.908769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519279345359619:2049] [node 2] ICP22 created new session: [6:7571519279345359711:2049]
2025-11-11T17:21:47.908820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519279345359711:2049] [node 2] ICS09 handshake done sender: [6:7571519279345359696:2115] self: [6:7571519279345359697:0] peer: [2:7571519278328374373:0] socket: 169 qp: -1
2025-11-11T17:21:47.908859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519279345359711:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:47.908942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:47.913176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519280779551675:2108] [node 5] ICP18 handshake (actor [4:7571519280779551703:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.918555Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:47.918262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359713:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.920260Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519279345359702:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.920398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519279345359669:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.921436Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519281632695219:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.921474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519281632695219:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:47.921016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519279345359713:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.921129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519279345359678:2093] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:47.921842Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519280779551686:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:47.921885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519280779551686:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.922610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359716:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.922896Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519281632695219:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:47.923291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519281632695209:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.923395Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519280779551686:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:47.923375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:47.923437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519281632695209:2110] [node 6] ICP22 created new session: [5:7571519281632695248:2110]
2025-11-11T17:21:47.923690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519281632695248:2110] [node 6] ICS09 handshake done sender: [5:7571519281632695219:2115] self: [5:7571519281632695218:0] peer: [6:7571519279345359715:0] socket: 141 qp: -1
2025-11-11T17:21:47.923736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519281632695248:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:47.923779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:47.925428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519280779551676:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:47.925495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:47.925540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519280779551676:2109] [node 6] ICP22 created new session: [4:7571519280779551714:2109]
2025-11-11T17:21:47.925579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519280779551714:2109] [node 6] ICS09 handshake done sender: [4:7571519280779551686:2114] self: [4:7571519280779551685:0] peer: [6:7571519279345359714:0] socket: 181 qp: -1
2025-11-11T17:21:47.925638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519280779551714:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:47.925671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:47.929140Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:47.931262Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519279345359713:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:47.931572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519279345359678:2093] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519279345359713:2122] held: no
2025-11-11T17:21:47.932166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519279345359678:2093] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.932274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:47.932321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519279345359678:2093] [node 5] ICP22 created new session: [6:7571519279345359718:2093]
2025-11-11T17:21:47.932392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519279345359718:2093] [node 5] ICS09 handshake done sender: [6:7571519279345359713:2122] self: [6:7571519279345359715:0] peer: [5:7571519281632695218:0] socket: 146 qp: -1
2025-11-11T17:21:47.932434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519279345359718:2093] [node 5] ICS10 traffic start
2025-11-11T17:21:47.932470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:47.933240Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:47.934918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519279345359717:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53263
2025-11-11T17:21:47.936025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519279345359669:2091] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519279345359702:2118] held: no
2025-11-11T17:21:47.936969Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519279345359702:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:47.937261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519279345359669:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:47.937365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:47.937413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519279345359669:2091] [node 4] ICP22 created new session: [6:7571519279345359720:2091]
2025-11-11T17:21:47.937461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519279345359720:2091] [node 4] ICS09 handshake done sender: [6:7571519279345359702:2118] self: [6:7571519279345359714:0] peer: [4:7571519280779551685:0] socket: 182 qp: -1
2025-11-11T17:21:47.937673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519279345359720:2091] [node 4] ICS10 traffic start
2025-11-11T17:21:47.937720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:47.997221Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:47.997477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519278328374398:2048] [node 1] ICS01 socket: 147 reason# EndOfStream
2025-11-11T17:21:47.997521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519278328374299:2048] [node 1] ICP30 unregister session Session# [2:7571519278328374398:2048] VirtualId# [2:7571519278328374393:0]
2025-11-11T17:21:47.997556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519278328374398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:47.997754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:47.997934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519282079780304:2048] [node 1] ICS01 socket: 145 reason# EndOfStream
2025-11-11T17:21:47.997971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519282079780201:2048] [node 1] ICP30 unregister session Session# [3:7571519282079780304:2048] VirtualId# [3:7571519282079780298:0]
2025-11-11T17:21:47.997995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519282079780304:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:47.998077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:47.998117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:47.998909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:47.999003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:48.000926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:48.001161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519280779551699:2048] [node 1] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-11T17:21:48.001199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519280779551699:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.001280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:48.001325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519280779551699:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:48.005266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:48.005574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519281632695242:2048] [node 1] ICS01 socket: 176 reason# EndOfStream
2025-11-11T17:21:48.005643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519281632695144:2048] [node 1] ICP30 unregister session Session# [5:7571519281632695242:2048] VirtualId# [5:7571519281632695235:0]
2025-11-11T17:21:48.005696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519281632695242:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.005783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:48.005827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:48.005266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:48.005560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519279345359705:2048] [node 1] ICS07 socket disconnect 159 reason# EndOfStream
2025-11-11T17:21:48.005622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519279345359705:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.005734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:48.005781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519279345359705:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:48.006847Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address
2025-11-11T17:21:48.007147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519280779551612:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:48.007198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519280779551699:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:48.007232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519280779551612:2048] [node 1] ICP30 unregister session Session# [4:7571519280779551699:2048] VirtualId# [4:7571519280779551696:0]
2025-11-11T17:21:48.007338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:48.007388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519280779551612:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.007407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.007991Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address
2025-11-11T17:21:48.009777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519279345359618:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:48.009833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519279345359705:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:48.009869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519279345359618:2048] [node 1] ICP30 unregister session Session# [6:7571519279345359705:2048] VirtualId# [6:7571519279345359700:0]
2025-11-11T17:21:48.009966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:48.009999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519279345359618:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.010013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.019932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519280779551612:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:48.020123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519279345359618:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:48.045006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:48.045286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519282079780296:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-11T17:21:48.045331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519282079780202:2049] [node 2] ICP30 unregister session Session# [3:7571519282079780296:2049] VirtualId# [3:7571519282079780290:0]
2025-11-11T17:21:48.045368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519282079780296:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.045486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:48.045531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:48.049287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:48.049557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519280779551707:2049] [node 2] ICS01 socket: 136 reason# EndOfStream
2025-11-11T17:21:48.049610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519280779551613:2049] [node 2] ICP30 unregister session Session# [4:7571519280779551707:2049] VirtualId# [4:7571519280779551694:0]
2025-11-11T17:21:48.049646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519280779551707:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.049754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:48.049798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:48.053751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:48.053995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519281632695228:2049] [node 2] ICS07 socket disconnect 158 reason# EndOfStream
2025-11-11T17:21:48.054026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519281632695228:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.054126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:48.054175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519281632695228:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:48.061366Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Couldn't connect to any resolved address
2025-11-11T17:21:48.060726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:48.061038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519279345359711:2049] [node 2] ICS01 socket: 169 reason# EndOfStream
2025-11-11T17:21:48.061120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519279345359619:2049] [node 2] ICP30 unregister session Session# [6:7571519279345359711:2049] VirtualId# [6:7571519279345359697:0]
2025-11-11T17:21:48.071769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519281632695145:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:48.061177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519279345359711:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.061340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:48.061417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:48.072778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519281632695228:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:48.072943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519281632695145:2049] [node 2] ICP30 unregister session Session# [5:7571519281632695228:2049] VirtualId# [5:7571519281632695226:0]
2025-11-11T17:21:48.073092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:48.073135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519281632695145:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.073161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.083293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519281632695145:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:48.134230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:48.134480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519279345359708:2050] [node 3] ICS01 socket: 177 reason# EndOfStream
2025-11-11T17:21:48.134519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519279345359620:2050] [node 3] ICP30 unregister session Session# [6:7571519279345359708:2050] VirtualId# [6:7571519279345359701:0]
2025-11-11T17:21:48.134547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519279345359708:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.134610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:48.134647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:48.124210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:48.124523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519280779551709:2050] [node 3] ICS07 socket disconnect 137 reason# EndOfStream
2025-11-11T17:21:48.124559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519280779551709:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.124650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:48.124688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519280779551709:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:48.125363Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2936) Couldn't connect to any resolved address
2025-11-11T17:21:48.135048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519280779551614:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2936) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:48.135088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519280779551709:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:48.135112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519280779551614:2050] [node 3] ICP30 unregister session Session# [4:7571519280779551709:2050] VirtualId# [4:7571519280779551695:0]
2025-11-11T17:21:48.135153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:48.135177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519280779551614:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2936) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.135203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2936) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:48.125600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:48.133785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519281632695245:2050] [node 3] ICS01 socket: 175 reason# EndOfStream
2025-11-11T17:21:48.133838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519281632695146:2050] [node 3] ICP30 unregister session Session# [5:7571519281632695245:2050] VirtualId# [5:7571519281632695236:0]
2025-11-11T17:21:48.133877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519281632695245:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.133993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:48.134042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:48.166408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:48.166597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519281632695240:2108] [node 4] ICS01 socket: 163 reason# EndOfStream
2025-11-11T17:21:48.166631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519281632695208:2108] [node 4] ICP30 unregister session Session# [5:7571519281632695240:2108] VirtualId# [5:7571519281632695231:0]
2025-11-11T17:21:48.166657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519281632695240:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.166741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:48.166779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:48.166534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:48.166755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519279345359720:2091] [node 4] ICS01 socket: 182 reason# EndOfStream
2025-11-11T17:21:48.166803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519279345359669:2091] [node 4] ICP30 unregister session Session# [6:7571519279345359720:2091] VirtualId# [6:7571519279345359714:0]
2025-11-11T17:21:48.166831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519279345359720:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.166922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:48.166961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:48.200379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:48.200732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519279345359718:2093] [node 5] ICS01 socket: 146 reason# EndOfStream
2025-11-11T17:21:48.200823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519279345359678:2093] [node 5] ICP30 unregister session Session# [6:7571519279345359718:2093] VirtualId# [6:7571519279345359715:0]
2025-11-11T17:21:48.200883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519279345359718:2093] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:48.201046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:48.201140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:48.302652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519283386751890:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:48.302718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519283386751891:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:48.324831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519286747436495:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:48.324881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519286747436496:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:48.353882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519283151603689:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:48.353930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519283151603690:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:48.372032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519283874482000:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:48.372071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519283874482001:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:48.372107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519283874482002:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:48.394628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519283090633039:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:48.394718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519283090633040:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:48.394805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519283090633041:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:48.415869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519283437257942:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:48.415919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519283437257943:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:48.416886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519283437257944:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:48.644904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519283437258003:2103] [node 4] ICP01 ready to work
2025-11-11T17:21:48.630508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519283151603752:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:48.650787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519283386751954:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:48.654924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519283151603752:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23756524ns) {Configure:104(23752351ns) {ProcessConfigured:135(9571552ns) {StartInitialHandshake:157(9568394ns) {DropHandshakes:469(2665ns) {DropIncomingHandshake:441(491ns) DropOutgoingHandshake:456(437ns)} GenerateSessionVirtualId:988(1052ns) PrepareNewSessionHandshake:476(5604ns) {SwitchToState:178(2847ns)}}}}}
2025-11-11T17:21:48.679528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519283386751953:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:48.657129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519283437258002:2104] [node 5] ICP01 ready to work
2025-11-11T17:21:48.655344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519283151603757:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:48.687965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519286747436558:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:48.683816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519283386751955:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:48.713758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519286747436559:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:48.688161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519283151603758:2110] [node 6] ICP01 ready to work
2025-11-11T17:21:48.724592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519283090633102:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:48.730922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519283874482063:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:48.723316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519286747436560:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:48.770391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519283151603689:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(74671322ns) {Configure:104(74665672ns) {ProcessConfigured:135(69536ns) {StartInitialHandshake:157(66685ns) {DropHandshakes:469(2573ns) {DropIncomingHandshake:441(576ns) DropOutgoingHandshake:456(505ns)} GenerateSessionVirtualId:988(1162ns) PrepareNewSessionHandshake:476(1634ns) {SwitchToState:178(472ns)}}}}}
2025-11-11T17:21:48.758273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519283437258003:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(71756285ns) {Configure:104(71744658ns) {ProcessConfigured:135(76280ns) {StartInitialHandshake:157(72419ns) {DropHandshakes:469(3564ns) {DropIncomingHandshake:441(831ns) DropOutgoingHandshake:456(563ns)} GenerateSessionVirtualId:988(1248ns) PrepareNewSessionHandshake:476(2262ns) {SwitchToState:178(723ns)}}}}}
2025-11-11T17:21:48.772266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519283090633103:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:48.788361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519283090633040:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22509179ns) {Configure:104(22500067ns) {ProcessConfigured:135(82312ns) {StartInitialHandshake:157(78683ns) {DropHandshakes:469(3479ns) {DropIncomingHandshake:441(840ns) DropOutgoingHandshake:456(698ns)} GenerateSessionVirtualId:988(1037ns) PrepareNewSessionHandshake:476(2615ns) {SwitchToState:178(733ns)}}}}}
2025-11-11T17:21:48.772498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519283874482064:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:48.803478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519286747436495:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(72636277ns) {Configure:104(58971614ns) {ProcessConfigured:135(77606ns) {StartInitialHandshake:157(74094ns) {DropHandshakes:469(3713ns) {DropIncomingHandshake:441(822ns) DropOutgoingHandshake:456(865ns)} GenerateSessionVirtualId:988(1124ns) PrepareNewSessionHandshake:476(2106ns) {SwitchToState:178(745ns)}}}}}
2025-11-11T17:21:48.810867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519283386751955:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(64231047ns) {Configure:104(64225546ns) {ProcessConfigured:135(65096ns) {StartInitialHandshake:157(62511ns) {DropHandshakes:469(980ns) {DropIncomingHandshake:441(259ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:476(2612ns) {SwitchToState:178(821ns)}}}}}
2025-11-11T17:21:48.814411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519283090633102:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17973587ns) {Configure:104(17959352ns) {ProcessConfigured:135(48254ns) {StartInitialHandshake:157(46404ns) {DropHandshakes:469(1433ns) {DropIncomingHandshake:441(302ns) DropOutgoingHandshake:456(663ns)} GenerateSessionVirtualId:988(308ns) PrepareNewSessionHandshake:476(1631ns) {SwitchToState:178(254ns)}}}}}
2025-11-11T17:21:48.824576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519283437258002:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(90201410ns) {Configure:104(90194550ns) {ProcessConfigured:135(2458593ns) {StartInitialHandshake:157(2456338ns) {DropHandshakes:469(498016ns) {DropIncomingHandshake:441(1693ns) DropOutgoingHandshake:456(864ns)} GenerateSessionVirtualId:988(2214ns) PrepareNewSessionHandshake:476(2723ns) {SwitchToState:178(141ns)}}}}}
2025-11-11T17:21:48.831755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519286747436560:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19867884ns) {Configure:104(19863540ns) {ProcessConfigured:135(228610ns) {StartInitialHandshake:157(226165ns) {DropHandshakes:469(6048ns) {DropIncomingHandshake:441(894ns) DropOutgoingHandshake:456(180ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:476(2949ns) {SwitchToState:178(1542ns)}}}}}
2025-11-11T17:21:48.831162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519283090633103:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37950680ns) {Configure:104(37947445ns) {ProcessConfigured:135(502123ns) {StartInitialHandshake:157(499402ns) {DropHandshakes:469(3273ns) {DropIncomingHandshake:441(768ns) DropOutgoingHandshake:456(483ns)} GenerateSessionVirtualId:988(1069ns) PrepareNewSessionHandshake:476(3165ns) {SwitchToState:178(951ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.843534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519286747436567:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19520
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.844160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519286747436571:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19520
2025-11-11T17:21:48.841771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519283386751954:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(27162935ns) {Configure:104(27156813ns) {ProcessConfigured:135(743369ns) {StartInitialHandshake:157(741816ns) {DropHandshakes:469(766ns) {DropIncomingHandshake:441(251ns) DropOutgoingHandshake:456(194ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:476(2329ns) {SwitchToState:178(926ns)}}}}}
2025-11-11T17:21:48.858030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519283437257942:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26550748ns) {Configure:104(26545803ns) {ProcessConfigured:135(8105928ns) {StartInitialHandshake:157(8099806ns) {DropHandshakes:469(4370ns) {DropIncomingHandshake:441(1346ns) DropOutgoingHandshake:456(684ns)} GenerateSessionVirtualId:988(2712ns) PrepareNewSessionHandshake:476(6066ns) {SwitchToState:178(3528ns)}}}}}
2025-11-11T17:21:48.861862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519283874482001:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(62273457ns) {Configure:104(62266034ns) {ProcessConfigured:135(16254175ns) {StartInitialHandshake:157(16250617ns) {DropHandshakes:469(2475ns) {DropIncomingHandshake:441(593ns) DropOutgoingHandshake:456(374ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:476(4375ns) {SwitchToState:178(1849ns)}}}}}
2025-11-11T17:21:48.859612Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436567:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.868868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519283151603768:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22076
2025-11-11T17:21:48.868598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519283874482064:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(52411681ns) {Configure:104(52407788ns) {ProcessConfigured:135(3872905ns) {StartInitialHandshake:157(3869408ns) {DropHandshakes:469(4292ns) {DropIncomingHandshake:441(1234ns) DropOutgoingHandshake:456(531ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:476(3072ns) {SwitchToState:178(1090ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.869744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519283386751953:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26181392ns) {Configure:104(26177117ns) {ProcessConfigured:135(871040ns) {StartInitialHandshake:157(868540ns) {DropHandshakes:469(2966ns) {DropIncomingHandshake:441(472ns) DropOutgoingHandshake:456(783ns)} GenerateSessionVirtualId:988(369ns) PrepareNewSessionHandshake:476(2468ns) {SwitchToState:178(837ns)}}}}}
2025-11-11T17:21:48.871246Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436571:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.872743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519283386751891:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(31181941ns) {Configure:104(31178102ns) {ProcessConfigured:135(741455ns) {StartInitialHandshake:157(738936ns) {DropHandshakes:469(1613ns) {DropIncomingHandshake:441(270ns) DropOutgoingHandshake:456(302ns)} GenerateSessionVirtualId:988(423ns) PrepareNewSessionHandshake:476(1591ns) {SwitchToState:178(616ns)}}}}}
2025-11-11T17:21:48.873555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519283090633117:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.878340Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519283090633117:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.878505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519283090633040:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.880384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519286747436578:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19520
2025-11-11T17:21:48.878655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519283386751963:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34370
2025-11-11T17:21:48.877771Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519283151603768:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.880510Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436566:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.878838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519283386751962:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34370
2025-11-11T17:21:48.879865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751963:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.880549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519286747436566:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.888776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519283874482069:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.884660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519283090633119:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-11T17:21:48.892275Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751962:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.892290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519283437257943:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23959654ns) {Configure:104(23952577ns) {ProcessConfigured:135(2349429ns) {StartInitialHandshake:157(467713ns) {DropHandshakes:469(2662ns) {DropIncomingHandshake:441(546ns) DropOutgoingHandshake:456(548ns)} GenerateSessionVirtualId:988(1191ns) PrepareNewSessionHandshake:476(1446ns) {SwitchToState:178(412ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.895552Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519286747436566:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:48.892656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519283874482063:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(27598384ns) {Configure:104(27593114ns) {ProcessConfigured:135(4968970ns) {StartInitialHandshake:157(4966166ns) {DropHandshakes:469(3352ns) {DropIncomingHandshake:441(630ns) DropOutgoingHandshake:456(612ns)} GenerateSessionVirtualId:988(512ns) PrepareNewSessionHandshake:476(2030ns) {SwitchToState:178(319ns)}}}}}
2025-11-11T17:21:48.901286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519283386751968:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.908832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519283386751973:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.909561Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519283090633119:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.909866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519283090633120:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-11T17:21:48.893651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519283874482070:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.918506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519283437258016:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.919022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519283437258020:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
2025-11-11T17:21:48.900554Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519283874482069:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.911502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519283874482002:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:48.917834Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519283874482070:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.911113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519283090633039:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.923810Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751959:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.912012Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519283090633117:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:48.933793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519283874482080:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
2025-11-11T17:21:48.923874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519283386751959:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:48.936694Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519283151603756:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.927227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519283090633040:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:48.939650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519286747436559:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:48.949730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519283437258016:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.939998Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751968:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.927307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:48.936750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519283151603756:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:48.939818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:48.933994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519283874482001:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:48.960064Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519283386751959:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:48.950341Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519283151603756:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:48.939853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519286747436559:2108] [node 5] ICP22 created new session: [2:7571519286747436581:2108]
2025-11-11T17:21:48.945869Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436564:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.927373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519283090633040:2049] [node 2] ICP22 created new session: [5:7571519283090633123:2049]
2025-11-11T17:21:48.975067Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751961:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.965800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519283437257944:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(60525788ns) {Configure:104(60520190ns) {ProcessConfigured:135(9852569ns) {StartInitialHandshake:157(9849085ns) {DropHandshakes:469(2259ns) {DropIncomingHandshake:441(289ns) DropOutgoingHandshake:456(679ns)} GenerateSessionVirtualId:988(657ns) PrepareNewSessionHandshake:476(3899ns) {SwitchToState:178(749ns)}}}}}
2025-11-11T17:21:48.945898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519286747436564:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:48.927430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519283090633123:2049] [node 2] ICS09 handshake done sender: [5:7571519283090633117:2115] self: [5:7571519283090633118:0] peer: [2:7571519286747436565:0] socket: 146 qp: -1
2025-11-11T17:21:48.949938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519286747436581:2108] [node 5] ICS09 handshake done sender: [2:7571519286747436566:2112] self: [2:7571519286747436565:0] peer: [5:7571519283090633118:0] socket: 145 qp: -1
2025-11-11T17:21:48.959100Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436578:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:48.959381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519286747436580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19520
2025-11-11T17:21:48.959881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519286747436495:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:48.962068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519286747436581:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:48.963872Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519286747436564:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:48.964668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:49.005925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519286747436558:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.005979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:49.006032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519286747436558:2107] [node 4] ICP22 created new session: [2:7571519291042403882:2107]
2025-11-11T17:21:49.006082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519291042403882:2107] [node 4] ICS09 handshake done sender: [2:7571519286747436564:2111] self: [2:7571519286747436563:0] peer: [4:7571519283874482082:0] socket: 144 qp: -1
2025-11-11T17:21:49.006117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519291042403882:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:49.006155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.994932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519283437257942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.975096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519283386751961:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:48.994986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519283151603752:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:48.927469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519283090633123:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:49.011772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519286747436559:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(64207458ns) {HandleHandshakeStatus:329(64204090ns) {DropIncomingHandshake:441(23401ns) DropOutgoingHandshake:456(534ns) SwitchToState:178(1119ns) ProcessPendingSessionEvents:495(546302ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 479341ns max# 84159ns}}}
2025-11-11T17:21:48.976043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519283386751954:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:48.927506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:48.965934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519283090633122:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-11T17:21:48.999665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.020477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519283090633040:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(66593619ns) {HandleHandshakeStatus:329(66587318ns) {DropIncomingHandshake:441(1006ns) DropOutgoingHandshake:456(7622ns) SwitchToState:178(1229ns) ProcessPendingSessionEvents:495(66136718ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 65995573ns max# 12822892ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.027277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519283437258023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
2025-11-11T17:21:49.034061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519287732225321:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:48.976111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.029802Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519283090633122:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.057744Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436580:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.049693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519283874482083:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
2025-11-11T17:21:49.003686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519283151603752:2107] [node 4] ICP22 created new session: [3:7571519283151603771:2107]
2025-11-11T17:21:49.043026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519283090633125:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-11T17:21:48.976165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519283386751954:2108] [node 5] ICP22 created new session: [1:7571519283386751978:2108]
2025-11-11T17:21:48.976212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519283386751978:2108] [node 5] ICS09 handshake done sender: [1:7571519283386751959:2111] self: [1:7571519283386751958:0] peer: [5:7571519283090633121:0] socket: 150 qp: -1
2025-11-11T17:21:48.976249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519283386751978:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:48.976291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:49.021836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519283386751975:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34370
2025-11-11T17:21:49.022065Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751957:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.022092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519283386751957:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.018894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519287446571068:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22076
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.057870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519291042403885:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19520
2025-11-11T17:21:49.053796Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519283874482080:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.045470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519287385600422:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-11T17:21:49.027849Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519283386751957:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:49.028216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519283386751955:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.028258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.019237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519283151603771:2107] [node 4] ICS09 handshake done sender: [3:7571519283151603756:2111] self: [3:7571519283151603755:0] peer: [4:7571519283874482081:0] socket: 133 qp: -1
2025-11-11T17:21:49.057506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519283874482084:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
2025-11-11T17:21:49.045686Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519283090633119:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:49.028328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519283386751955:2109] [node 6] ICP22 created new session: [1:7571519287681719279:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.075460Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519283437258023:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.062060Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519283874482069:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:49.022970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519283151603771:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:49.028378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519287681719279:2109] [node 6] ICS09 handshake done sender: [1:7571519283386751957:2110] self: [1:7571519283386751956:0] peer: [6:7571519283437258024:0] socket: 149 qp: -1
2025-11-11T17:21:49.088608Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519286747436578:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:49.046409Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519287385600422:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.076426Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519283437258020:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.089840Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436569:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.023026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:49.028693Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519283386751961:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:49.089875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519286747436569:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.072930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519283874482002:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.076783Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519283437258016:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:49.067901Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519283151603763:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.067942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519283151603763:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.071528Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519283151603763:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:49.094193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519283151603757:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.095918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:49.101138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519283151603757:2109] [node 5] ICP22 created new session: [3:7571519287446571072:2109]
2025-11-11T17:21:49.046490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519283090633039:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.029747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519287681719279:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:49.091157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519286747436558:2107] [node 4] ICP18 handshake (actor [2:7571519286747436580:2118]) is too old
2025-11-11T17:21:49.077184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519283437257942:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.072987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:49.091211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519286747436495:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.091256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:49.091301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519286747436495:2048] [node 1] ICP22 created new session: [2:7571519291042403887:2048]
2025-11-11T17:21:49.091347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519291042403887:2048] [node 1] ICS09 handshake done sender: [2:7571519286747436578:2117] self: [2:7571519286747436584:0] peer: [1:7571519283386751960:0] socket: 153 qp: -1
2025-11-11T17:21:49.091396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519291042403887:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:49.091429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:49.038307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:49.110020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519287446571072:2109] [node 5] ICS09 handshake done sender: [3:7571519283151603763:2113] self: [3:7571519283151603762:0] peer: [5:7571519287385600427:0] socket: 178 qp: -1
2025-11-11T17:21:49.073039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519283874482002:2050] [node 3] ICP22 created new session: [4:7571519288169449382:2050]
2025-11-11T17:21:49.076236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519283386751890:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.076308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:49.076365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519283386751890:2048] [node 2] ICP22 created new session: [1:7571519287681719281:2048]
2025-11-11T17:21:49.079669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519287681719281:2048] [node 2] ICS09 handshake done sender: [1:7571519283386751961:2112] self: [1:7571519283386751960:0] peer: [2:7571519286747436584:0] socket: 152 qp: -1
2025-11-11T17:21:49.088376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519287681719281:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:49.088428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:49.122731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519283386751955:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(90897298ns) {HandleHandshakeStatus:329(90893015ns) {DropIncomingHandshake:441(694ns) DropOutgoingHandshake:456(951ns) SwitchToState:178(778ns) ProcessPendingSessionEvents:495(53572570ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 53462012ns max# 8041267ns}}}
2025-11-11T17:21:49.122931Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751975:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.046546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:49.077224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:49.077251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519283437257942:2048] [node 1] ICP22 created new session: [6:7571519287732225323:2048]
2025-11-11T17:21:49.077280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519287732225323:2048] [node 1] ICS09 handshake done sender: [6:7571519283437258016:2115] self: [6:7571519283437258024:0] peer: [1:7571519283386751956:0] socket: 156 qp: -1
2025-11-11T17:21:49.077310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519287732225323:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:49.077333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:49.078525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519283437257943:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:49.078604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519283437258003:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:49.085724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519287732225322:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.117734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519287446571072:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:49.046593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519283090633039:2048] [node 1] ICP22 created new session: [5:7571519287385600425:2048]
2025-11-11T17:21:49.073087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519288169449382:2050] [node 3] ICS09 handshake done sender: [4:7571519283874482069:2112] self: [4:7571519283874482081:0] peer: [3:7571519283151603755:0] socket: 148 qp: -1
2025-11-11T17:21:49.121814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:49.046639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519287385600425:2048] [node 1] ICS09 handshake done sender: [5:7571519283090633119:2116] self: [5:7571519283090633121:0] peer: [1:7571519283386751958:0] socket: 151 qp: -1
2025-11-11T17:21:49.146232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519291042403886:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19520
2025-11-11T17:21:49.073126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519288169449382:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.168725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519287732225328:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
2025-11-11T17:21:49.073192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:49.046673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519287385600425:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:49.169042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519287732225322:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.046738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:49.084571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519288169449381:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
2025-11-11T17:21:49.187518Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519286747436569:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:49.051032Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:49.186772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519283151603757:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(96538780ns) {HandleHandshakeStatus:329(96534686ns) {DropIncomingHandshake:441(18947ns) DropOutgoingHandshake:456(537ns) SwitchToState:178(2412ns) ProcessPendingSessionEvents:495(64023116ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 63916298ns max# 11699372ns}}}
2025-11-11T17:21:49.186982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519287446571068:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.193698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519283437258002:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.211286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519283151603690:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:49.054339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519283090633041:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:49.213845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519283151603752:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(198897184ns) {HandleHandshakeStatus:329(198893326ns) {DropIncomingHandshake:441(657ns) DropOutgoingHandshake:456(1274ns) SwitchToState:178(353ns) ProcessPendingSessionEvents:495(162806045ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 162610467ns max# 36785961ns}}}
2025-11-11T17:21:49.085288Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519283874482070:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:49.206339Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519283437258023:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:49.067524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519283090633039:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:49.076425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519287385600429:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-11T17:21:49.076816Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519287385600422:2120] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:49.085014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519283090633041:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.088282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:49.088752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519283090633041:2050] [node 3] ICP22 created new session: [5:7571519287385600430:2050]
2025-11-11T17:21:49.089453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519287385600430:2050] [node 3] ICS09 handshake done sender: [5:7571519287385600422:2120] self: [5:7571519287385600427:0] peer: [3:7571519283151603762:0] socket: 182 qp: -1
2025-11-11T17:21:49.097713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519287385600430:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:49.097785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:49.146548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519283090633041:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(21571115ns) {HandleHandshakeStatus:329(21566874ns) {DropIncomingHandshake:441(797ns) DropOutgoingHandshake:456(7001ns) SwitchToState:178(473ns) ProcessPendingSessionEvents:495(602469ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 544725ns max# 84673ns}}}
2025-11-11T17:21:49.199413Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519283090633113:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.199445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519283090633113:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.218562Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519283090633113:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:49.232952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519283090633103:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.233009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:49.234328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519283090633103:2109] [node 6] ICP22 created new session: [5:7571519287385600433:2109]
2025-11-11T17:21:49.222887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519286747436560:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.243241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751973:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.243362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519283386751955:2109] [node 6] ICP18 handshake (actor [1:7571519283386751975:2119]) is too old
2025-11-11T17:21:49.114744Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519283874482068:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.230190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519283437257943:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.247942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519287385600433:2109] [node 6] ICS09 handshake done sender: [5:7571519283090633113:2114] self: [5:7571519283090633112:0] peer: [6:7571519287732225329:0] socket: 137 qp: -1
2025-11-11T17:21:49.222940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.230236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:49.114774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519283874482068:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.249063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519287446571076:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22076
2025-11-11T17:21:49.222992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519286747436560:2109] [node 6] ICP22 created new session: [2:7571519291042403890:2109]
2025-11-11T17:21:49.249697Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519287446571068:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:49.276374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519283151603690:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.276424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:49.276478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519283151603690:2049] [node 2] ICP22 created new session: [3:7571519287446571078:2049]
2025-11-11T17:21:49.276521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519287446571078:2049] [node 2] ICS09 handshake done sender: [3:7571519287446571068:2116] self: [3:7571519287446571075:0] peer: [2:7571519286747436570:0] socket: 177 qp: -1
2025-11-11T17:21:49.115784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519283874482001:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.257446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519287385600433:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:49.230294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519283437257943:2049] [node 2] ICP22 created new session: [6:7571519287732225333:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.223045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519291042403890:2109] [node 6] ICS09 handshake done sender: [2:7571519286747436569:2114] self: [2:7571519286747436568:0] peer: [6:7571519287732225325:0] socket: 173 qp: -1
2025-11-11T17:21:49.230338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519287732225333:2049] [node 2] ICS09 handshake done sender: [6:7571519283437258023:2118] self: [6:7571519287732225325:0] peer: [2:7571519286747436568:0] socket: 174 qp: -1
2025-11-11T17:21:49.276562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519287446571078:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:49.115900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:49.276109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:49.230376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519287732225333:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:49.230403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:49.223078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519291042403890:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:49.276597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:49.115959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519283874482001:2049] [node 2] ICP22 created new session: [4:7571519288169449386:2049]
2025-11-11T17:21:49.247564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519287732225330:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
2025-11-11T17:21:49.270625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519287732225331:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
2025-11-11T17:21:49.271499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519287732225332:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
2025-11-11T17:21:49.272070Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519287732225322:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:49.272767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519283437258002:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.272811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:49.272860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519283437258002:2104] [node 5] ICP22 created new session: [6:7571519287732225336:2104]
2025-11-11T17:21:49.272905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519287732225336:2104] [node 5] ICS09 handshake done sender: [6:7571519287732225322:2120] self: [6:7571519287732225329:0] peer: [5:7571519283090633112:0] socket: 188 qp: -1
2025-11-11T17:21:49.272943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519287732225336:2104] [node 5] ICS10 traffic start
2025-11-11T17:21:49.272983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:49.273936Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519283437258020:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:49.294221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519283437258003:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.294312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:49.294369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519283437258003:2103] [node 4] ICP22 created new session: [6:7571519287732225338:2103]
2025-11-11T17:21:49.294420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519287732225338:2103] [node 4] ICS09 handshake done sender: [6:7571519283437258020:2116] self: [6:7571519287732225326:0] peer: [4:7571519283874482067:0] socket: 167 qp: -1
2025-11-11T17:21:49.294457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519287732225338:2103] [node 4] ICS10 traffic start
2025-11-11T17:21:49.294494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:49.295707Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519287732225331:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.306621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519283437257944:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:49.288280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519283090633103:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(54870630ns) {HandleHandshakeStatus:329(54868459ns) {DropIncomingHandshake:441(12888ns) DropOutgoingHandshake:456(476ns) SwitchToState:178(373ns) ProcessPendingSessionEvents:495(6605402ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6542789ns max# 6061155ns}}}
2025-11-11T17:21:49.223114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:49.116016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519288169449386:2049] [node 2] ICS09 handshake done sender: [4:7571519283874482070:2113] self: [4:7571519283874482082:0] peer: [2:7571519286747436563:0] socket: 154 qp: -1
2025-11-11T17:21:49.336226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519283386751890:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(246353159ns) {HandleHandshakeStatus:329(246350243ns) {DropIncomingHandshake:441(15823ns) DropOutgoingHandshake:456(564ns) SwitchToState:178(307ns) ProcessPendingSessionEvents:495(228495814ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 221887326ns max# 14980751ns}}}
2025-11-11T17:21:49.116052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519288169449386:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:49.116097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:49.139938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519283874482002:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(66146985ns) {HandleHandshakeStatus:329(66144264ns) {DropIncomingHandshake:441(1002ns) DropOutgoingHandshake:456(5766ns) SwitchToState:178(924ns) ProcessPendingSessionEvents:495(65698525ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 58031934ns max# 7723707ns}}}
2025-11-11T17:21:49.289047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519287385600435:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.224146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519286747436575:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.318016Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519287385600435:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.318357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519283090633102:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.224168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519286747436575:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.229171Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519291042403886:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.229312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519286747436560:2109] [node 6] ICP18 handshake (actor [2:7571519291042403886:2120]) is too old
2025-11-11T17:21:49.337722Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751965:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.337763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519283386751965:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.140081Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:49.220986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519283874482001:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:49.221124Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519288169449381:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.221627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519288169449385:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
2025-11-11T17:21:49.232603Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519283874482068:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:49.279930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519283874482064:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.280048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:49.229788Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519286747436575:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:49.249555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519286747436496:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.280098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519283874482064:2109] [node 6] ICP22 created new session: [4:7571519288169449391:2109]
2025-11-11T17:21:49.340386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519287446571077:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22076
2025-11-11T17:21:49.340448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519287385600437:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-11T17:21:49.280155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519288169449391:2109] [node 6] ICS09 handshake done sender: [4:7571519283874482068:2111] self: [4:7571519283874482067:0] peer: [6:7571519287732225326:0] socket: 161 qp: -1
2025-11-11T17:21:49.257732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:49.341130Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519283151603767:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.280189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519288169449391:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:49.357515Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519283386751965:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:49.276225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519286747436496:2049] [node 3] ICP22 created new session: [2:7571519291042403893:2049]
2025-11-11T17:21:49.362378Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519287385600435:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:49.341152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519283151603767:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.280252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:49.366077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519283386751953:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.342112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519287446571080:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22076
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.305465Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519288169449385:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.362903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519283090633102:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.278877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519291042403893:2049] [node 3] ICS09 handshake done sender: [2:7571519286747436575:2116] self: [2:7571519286747436570:0] peer: [3:7571519287446571075:0] socket: 175 qp: -1
2025-11-11T17:21:49.368448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:49.362950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:49.350753Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519283151603767:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:49.297655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519291042403893:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:49.323213Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519283874482076:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.394616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519287732225341:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16152
2025-11-11T17:21:49.395161Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519287732225331:2123] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:49.363529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519283090633102:2108] [node 4] ICP22 created new session: [5:7571519287385600438:2108]
2025-11-11T17:21:49.369556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519283386751953:2107] [node 4] ICP22 created new session: [1:7571519287681719286:2107]
2025-11-11T17:21:49.369139Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519287446571080:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.378512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519287681719286:2107] [node 4] ICS09 handshake done sender: [1:7571519283386751965:2115] self: [1:7571519283386751964:0] peer: [4:7571519288169449394:0] socket: 187 qp: -1
2025-11-11T17:21:49.385906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519287681719286:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:49.385964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:49.399285Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519283386751967:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.399327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519283386751967:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.323246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519283874482076:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:49.336225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519283874482000:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:49.340454Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519283874482076:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:49.357100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519283874482064:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(57450228ns) {HandleHandshakeStatus:329(57448074ns) {DropIncomingHandshake:441(12783ns) DropOutgoingHandshake:456(405ns) SwitchToState:178(426ns) ProcessPendingSessionEvents:495(57034240ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56917459ns max# 10358019ns}}}
2025-11-11T17:21:49.357449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519283874482063:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.358466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:49.358526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519283874482063:2108] [node 5] ICP22 created new session: [4:7571519288169449395:2108]
2025-11-11T17:21:49.358575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519288169449395:2108] [node 5] ICS09 handshake done sender: [4:7571519283874482076:2116] self: [4:7571519283874482071:0] peer: [5:7571519287385600436:0] socket: 192 qp: -1
2025-11-11T17:21:49.358609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519288169449395:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:49.358674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:49.373644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519288169449397:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43554
2025-11-11T17:21:49.297713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:49.395491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519283437257944:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:49.363569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519287385600438:2108] [node 4] ICS09 handshake done sender: [5:7571519287385600435:2122] self: [5:7571519287385600436:0] peer: [4:7571519283874482071:0] socket: 169 qp: -1
2025-11-11T17:21:49.369254Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519287446571077:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:49.369309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519283151603758:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.369342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:49.369384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519283151603758:2110] [node 6] ICP22 created new session: [3:7571519287446571082:2110]
2025-11-11T17:21:49.369435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519287446571082:2110] [node 6] ICS09 handshake done sender: [3:7571519283151603767:2114] self: [3:7571519283151603766:0] peer: [6:7571519287732225340:0] socket: 139 qp: -1
2025-11-11T17:21:49.369464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519287446571082:2110] [node 6] ICS10 traffic start
2025-11-11T17:21:49.374145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:49.390675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519283151603690:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(113379473ns) {HandleHandshakeStatus:329(113376847ns) {DropIncomingHandshake:441(695ns) DropOutgoingHandshake:456(4644ns) SwitchToState:178(839ns) ProcessPendingSessionEvents:495(113029055ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 103666517ns max# 10363762ns}}}
2025-11-11T17:21:49.392295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519283151603689:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:49.403604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519287446571085:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22076
2025-11-11T17:21:49.404111Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519287446571077:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:49.402822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519283386751967:2116] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:49.405410Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:49.363602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519287385600438:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:49.320997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519286747436496:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(70440752ns) {HandleHandshakeStatus:329(70438158ns) {DropIncomingHandshake:441(8111952ns) DropOutgoingHandshake:456(1778ns) SwitchToState:178(1328ns) ProcessPendingSessionEvents:495(11622336ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11555864ns max# 11119741ns}}}
2025-11-11T17:21:49.408417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519283386751953:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(41504622ns) {HandleHandshakeStatus:329(41502658ns) {DropIncomingHandshake:441(14254ns) DropOutgoingHandshake:456(478ns) SwitchToState:178(635ns) ProcessPendingSessionEvents:495(7145133ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7079573ns max# 5404153ns}}}
2025-11-11T17:21:49.408949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519283151603689:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.409004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:49.409057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519283151603689:2048] [node 1] ICP22 created new session: [3:7571519287446571086:2048]
2025-11-11T17:21:49.409100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519287446571086:2048] [node 1] ICS09 handshake done sender: [3:7571519287446571077:2118] self: [3:7571519287446571084:0] peer: [1:7571519283386751966:0] socket: 189 qp: -1
2025-11-11T17:21:49.409136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519287446571086:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:49.409169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:49.414447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519283386751891:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:49.414520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:49.414565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519283386751891:2049] [node 3] ICP22 created new session: [1:7571519287681719289:2049]
2025-11-11T17:21:49.414615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519287681719289:2049] [node 3] ICS09 handshake done sender: [1:7571519283386751967:2116] self: [1:7571519283386751966:0] peer: [3:7571519287446571084:0] socket: 168 qp: -1
2025-11-11T17:21:49.414682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519287681719289:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:49.414746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:49.395551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:49.363641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:49.395600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519283437257944:2050] [node 3] ICP22 created new session: [6:7571519287732225342:2050]
2025-11-11T17:21:49.428009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519283874482000:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519288169449385:2121] held: no
2025-11-11T17:21:49.395903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519287732225342:2050] [node 3] ICS09 handshake done sender: [6:7571519287732225331:2123] self: [6:7571519287732225340:0] peer: [3:7571519283151603766:0] socket: 147 qp: -1
2025-11-11T17:21:49.402937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519287732225342:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:49.403023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:49.430944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519288169449385:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:49.432336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519283874482000:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:49.432420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:49.432476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519283874482000:2048] [node 1] ICP22 created new session: [4:7571519288169449398:2048]
2025-11-11T17:21:49.432527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519288169449398:2048] [node 1] ICS09 handshake done sender: [4:7571519288169449385:2121] self: [4:7571519288169449394:0] peer: [1:7571519283386751964:0] socket: 191 qp: -1
2025-11-11T17:21:49.432560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519288169449398:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:49.432621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:49.435988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519283151603758:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(62235353ns) {HandleHandshakeStatus:329(62233231ns) {DropIncomingHandshake:441(351ns) DropOutgoingHandshake:456(569ns) SwitchToState:178(445ns) ProcessPendingSessionEvents:495(45328129ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 38483207ns max# 11167769ns}}}
2025-11-11T17:21:49.436574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519283151603758:2110] [node 6] ICP18 handshake (actor [3:7571519287446571080:2119]) is too old
2025-11-11T17:21:49.480367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:49.480638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519291042403887:2048] [node 1] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:21:49.480684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519286747436495:2048] [node 1] ICP30 unregister session Session# [2:7571519291042403887:2048] VirtualId# [2:7571519286747436584:0]
2025-11-11T17:21:49.480721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519291042403887:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.480824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:49.480872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:49.483733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:49.483992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519287446571086:2048] [node 1] ICS01 socket: 189 reason# EndOfStream
2025-11-11T17:21:49.484024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519283151603689:2048] [node 1] ICP30 unregister session Session# [3:7571519287446571086:2048] VirtualId# [3:7571519287446571084:0]
2025-11-11T17:21:49.484059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519287446571086:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.484169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:49.484214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:49.489788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:49.490021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519288169449398:2048] [node 1] ICS01 socket: 191 reason# EndOfStream
2025-11-11T17:21:49.490062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519283874482000:2048] [node 1] ICP30 unregister session Session# [4:7571519288169449398:2048] VirtualId# [4:7571519288169449394:0]
2025-11-11T17:21:49.490100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519288169449398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.490190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:49.490232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:49.490946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:49.491136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519287732225323:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-11T17:21:49.491172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519283437257942:2048] [node 1] ICP30 unregister session Session# [6:7571519287732225323:2048] VirtualId# [6:7571519283437258024:0]
2025-11-11T17:21:49.491198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519287732225323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.497653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:49.491238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:49.499781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:49.501507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519287385600425:2048] [node 1] ICS01 socket: 151 reason# EndOfStream
2025-11-11T17:21:49.501556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519283090633039:2048] [node 1] ICP30 unregister session Session# [5:7571519287385600425:2048] VirtualId# [5:7571519283090633121:0]
2025-11-11T17:21:49.501614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519287385600425:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.501732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:49.501788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:49.594588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:49.594809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519287446571078:2049] [node 2] ICS07 socket disconnect 177 reason# EndOfStream
2025-11-11T17:21:49.594843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519287446571078:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.594940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:49.594979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519287446571078:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:49.599383Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:21:49.599560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519283151603690:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:49.599599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519287446571078:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:49.601580Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address
2025-11-11T17:21:49.601894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519283151603690:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:49.601961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519287446571078:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:49.602002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519283151603690:2049] [node 2] ICP30 unregister session Session# [3:7571519287446571078:2049] VirtualId# [3:7571519287446571075:0]
2025-11-11T17:21:49.602112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:49.602143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519283151603690:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.602159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.602857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:49.603336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519288169449386:2049] [node 2] ICS07 socket disconnect 154 reason# EndOfStream
2025-11-11T17:21:49.603496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519288169449386:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.603577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:49.609800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:49.610031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519283090633123:2049] [node 2] ICS01 socket: 146 reason# EndOfStream
2025-11-11T17:21:49.610076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519283090633040:2049] [node 2] ICP30 unregister session Session# [5:7571519283090633123:2049] VirtualId# [5:7571519283090633118:0]
2025-11-11T17:21:49.610115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519283090633123:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.610219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:49.610266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:49.610375Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:49.610558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519287732225333:2049] [node 2] ICS01 socket: 174 reason# EndOfStream
2025-11-11T17:21:49.610591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519283437257943:2049] [node 2] ICP30 unregister session Session# [6:7571519287732225333:2049] VirtualId# [6:7571519287732225325:0]
2025-11-11T17:21:49.610615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519287732225333:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.610666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:49.610704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:49.603797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519288169449386:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:49.612704Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address
2025-11-11T17:21:49.612993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519283874482001:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:49.613047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519288169449386:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:49.613080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519283874482001:2049] [node 2] ICP30 unregister session Session# [4:7571519288169449386:2049] VirtualId# [4:7571519283874482082:0]
2025-11-11T17:21:49.613173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:49.613210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519283874482001:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.613248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.613675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519283151603690:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:49.625976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519283874482001:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:49.661850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:49.662081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519288169449382:2050] [node 3] ICS07 socket disconnect 148 reason# EndOfStream
2025-11-11T17:21:49.662112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519288169449382:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.662201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:49.662245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519288169449382:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:49.662773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:49.662947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519287385600430:2050] [node 3] ICS01 socket: 182 reason# EndOfStream
2025-11-11T17:21:49.662979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519283090633041:2050] [node 3] ICP30 unregister session Session# [5:7571519287385600430:2050] VirtualId# [5:7571519287385600427:0]
2025-11-11T17:21:49.663013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519287385600430:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.663072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:49.663115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:49.663212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:49.663395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519287732225342:2050] [node 3] ICS01 socket: 147 reason# EndOfStream
2025-11-11T17:21:49.663441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519283437257944:2050] [node 3] ICP30 unregister session Session# [6:7571519287732225342:2050] VirtualId# [6:7571519287732225340:0]
2025-11-11T17:21:49.663466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519287732225342:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.663520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:49.663554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:49.664964Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15446) Couldn't connect to any resolved address
2025-11-11T17:21:49.665214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519283874482002:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15446) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:49.665265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519288169449382:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:49.665297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519283874482002:2050] [node 3] ICP30 unregister session Session# [4:7571519288169449382:2050] VirtualId# [4:7571519283874482081:0]
2025-11-11T17:21:49.665378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:49.665410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519283874482002:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15446) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.665423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15446) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.677697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519283874482002:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:49.725722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:49.726023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519287385600438:2108] [node 4] ICS01 socket: 169 reason# EndOfStream
2025-11-11T17:21:49.726065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519283090633102:2108] [node 4] ICP30 unregister session Session# [5:7571519287385600438:2108] VirtualId# [5:7571519287385600436:0]
2025-11-11T17:21:49.726112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519287385600438:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.726197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:49.726241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:49.733923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:49.734460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519287732225338:2103] [node 4] ICS01 socket: 167 reason# EndOfStream
2025-11-11T17:21:49.734501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519283437258003:2103] [node 4] ICP30 unregister session Session# [6:7571519287732225338:2103] VirtualId# [6:7571519287732225326:0]
2025-11-11T17:21:49.734535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519287732225338:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.734631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:49.734679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:49.761776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:49.762018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519287732225336:2104] [node 5] ICS07 socket disconnect 188 reason# EndOfStream
2025-11-11T17:21:49.762048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519287732225336:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:49.762152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:49.762197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519287732225336:2104] [node 5] ICS15 start handshake
2025-11-11T17:21:49.763045Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7988) Couldn't connect to any resolved address
2025-11-11T17:21:49.763299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519283437258002:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7988) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:49.763361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519287732225336:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:49.763400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519283437258002:2104] [node 5] ICP30 unregister session Session# [6:7571519287732225336:2104] VirtualId# [6:7571519287732225329:0]
2025-11-11T17:21:49.763510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:21:49.763544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519283437258002:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.763558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:49.777694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519283437258002:2104] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:49.845271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519289033441693:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:49.845425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519289033441692:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:49.866324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519287709904094:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:49.866368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519287709904095:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:49.889480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519289577268505:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:49.889522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519289577268506:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:49.920447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519288615753084:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:49.920451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519288615753083:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:49.920494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519288615753085:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:49.941403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519290704375188:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:49.941449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519290704375189:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:49.941484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519290704375190:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:49.964468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519286980855890:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:49.964512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519286980855892:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:49.964772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519286980855891:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:50.126143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519293328409049:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:50.129954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519291275823246:2104] [node 5] ICP01 ready to work
2025-11-11T17:21:50.229146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519293328409053:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:50.265891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519292004871456:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:50.266145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519293872235882:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:50.257683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519292910720444:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:50.261755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519294999342547:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:50.260698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519293328409054:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:50.273638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519292004871455:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:50.285885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519292910720445:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:50.305785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519294999342548:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:50.339125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519288615753084:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(89050610ns) {Configure:104(89046137ns) {ProcessConfigured:135(75932ns) {StartInitialHandshake:157(70347ns) {DropHandshakes:469(3123ns) {DropIncomingHandshake:441(543ns) DropOutgoingHandshake:456(623ns)} GenerateSessionVirtualId:988(1067ns) PrepareNewSessionHandshake:476(2089ns) {SwitchToState:178(665ns)}}}}}
2025-11-11T17:21:50.301947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519293872235883:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:50.285635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519292004871457:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:50.350014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519293328409049:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(223537291ns) {Configure:104(223532088ns) {ProcessConfigured:135(4265126ns) {StartInitialHandshake:157(1526641ns) {DropHandshakes:469(2999ns) {DropIncomingHandshake:441(782ns) DropOutgoingHandshake:456(309ns)} GenerateSessionVirtualId:988(1410ns) PrepareNewSessionHandshake:476(2561ns) {SwitchToState:178(979ns)}}}}}
2025-11-11T17:21:50.323015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519291275823259:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:50.321653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519293872235884:2114] [node 6] ICP01 ready to work
2025-11-11T17:21:50.357783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519289577268505:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(91101906ns) {Configure:104(91097269ns) {ProcessConfigured:135(8884566ns) {StartInitialHandshake:157(8882137ns) {DropHandshakes:469(853ns) {DropIncomingHandshake:441(243ns) DropOutgoingHandshake:456(162ns)} GenerateSessionVirtualId:988(328ns) PrepareNewSessionHandshake:476(5031ns) {SwitchToState:178(1981ns)}}}}}
2025-11-11T17:21:50.363165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519289033441692:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(73196800ns) {Configure:104(73187387ns) {ProcessConfigured:135(4058403ns) {StartInitialHandshake:157(4056204ns) {DropHandshakes:469(4077ns) {DropIncomingHandshake:441(1360ns) DropOutgoingHandshake:456(682ns)} GenerateSessionVirtualId:988(831ns) PrepareNewSessionHandshake:476(11233ns) {SwitchToState:178(1815ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.396821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519291275823246:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(94703864ns) {Configure:104(94697187ns) {ProcessConfigured:135(72589ns) {StartInitialHandshake:157(70176ns) {DropHandshakes:469(4157ns) {DropIncomingHandshake:441(937ns) DropOutgoingHandshake:456(707ns)} GenerateSessionVirtualId:988(1094ns) PrepareNewSessionHandshake:476(2396ns) {SwitchToState:178(873ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.418239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519291275823259:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16212223ns) {Configure:104(16206538ns) {ProcessConfigured:135(62740ns) {StartInitialHandshake:157(60200ns) {DropHandshakes:469(4144ns) {DropIncomingHandshake:441(928ns) DropOutgoingHandshake:456(493ns)} GenerateSessionVirtualId:988(1679ns) PrepareNewSessionHandshake:476(2399ns) {SwitchToState:178(839ns)}}}}}
2025-11-11T17:21:50.428695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519293872235889:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16726
2025-11-11T17:21:50.413422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519292004871458:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3136
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.451462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720450:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
2025-11-11T17:21:50.451893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720455:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.420857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342561:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
2025-11-11T17:21:50.449627Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235889:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.471269Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519292910720450:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.454547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342563:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
2025-11-11T17:21:50.471935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519290704375190:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(107474500ns) {Configure:104(107469938ns) {ProcessConfigured:135(30490507ns) {StartInitialHandshake:157(30486832ns) {DropHandshakes:469(1607ns) {DropIncomingHandshake:441(366ns) DropOutgoingHandshake:456(189ns)} GenerateSessionVirtualId:988(530ns) PrepareNewSessionHandshake:476(5482ns) {SwitchToState:178(2941ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.475301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823264:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.478365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519289033441693:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(125981479ns) {Configure:104(125976233ns) {ProcessConfigured:135(81464ns) {StartInitialHandshake:157(77878ns) {DropHandshakes:469(6728ns) {DropIncomingHandshake:441(708ns) DropOutgoingHandshake:456(941ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:476(2305ns) {SwitchToState:178(800ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.413787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519292004871463:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3136
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.466136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519293872235882:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(63909007ns) {Configure:104(63904854ns) {ProcessConfigured:135(89571ns) {StartInitialHandshake:157(86306ns) {DropHandshakes:469(3993ns) {DropIncomingHandshake:441(741ns) DropOutgoingHandshake:456(633ns)} GenerateSessionVirtualId:988(986ns) PrepareNewSessionHandshake:476(2150ns) {SwitchToState:178(659ns)}}}}}
2025-11-11T17:21:50.490064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519288615753085:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(141686716ns) {Configure:104(141683044ns) {ProcessConfigured:135(66141ns) {StartInitialHandshake:157(63519ns) {DropHandshakes:469(4002ns) {DropIncomingHandshake:441(702ns) DropOutgoingHandshake:456(940ns)} GenerateSessionVirtualId:988(1102ns) PrepareNewSessionHandshake:476(1897ns) {SwitchToState:178(742ns)}}}}}
2025-11-11T17:21:50.506172Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519292910720455:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.506710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519288615753084:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.506801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.472066Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871458:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.520877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519291275823264:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.484300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519293872235898:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16726
2025-11-11T17:21:50.492154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519293328409058:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28236
2025-11-11T17:21:50.517684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519294999342561:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.517902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519290704375189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.521400Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519294999342563:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.521559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519294999342547:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.480933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519292004871470:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3136
2025-11-11T17:21:50.548590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519286980855892:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.534173Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519292910720454:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.513254Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235898:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.492315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519293328409057:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28236
2025-11-11T17:21:50.569844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519286980855892:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(24569506ns) {IncomingHandshake:261(17582430ns) {DropIncomingHandshake:441(946ns) PrepareNewSessionHandshake:476(2136ns) {SwitchToState:178(1091ns)} GenerateSessionVirtualId:988(1086ns) IssueIncomingHandshakeReply:193(17532186ns) {SwitchToState:178(5318ns)}}}
2025-11-11T17:21:50.480979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519292004871471:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3136
2025-11-11T17:21:50.513508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519293872235901:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16726
2025-11-11T17:21:50.534210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519292910720454:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.492526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519293328409063:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28236
2025-11-11T17:21:50.481240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519292004871455:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(159642557ns) {Configure:104(159636899ns) {ProcessConfigured:135(11332948ns) {StartInitialHandshake:157(11328665ns) {DropHandshakes:469(2217714ns) {DropIncomingHandshake:441(604ns) DropOutgoingHandshake:456(1860ns)} GenerateSessionVirtualId:988(2521ns) PrepareNewSessionHandshake:476(2543ns) {SwitchToState:178(942ns)}}}}}
2025-11-11T17:21:50.558331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342566:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.519662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519292004871474:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3136
2025-11-11T17:21:50.492576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519293328409064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28236
2025-11-11T17:21:50.539825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720464:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
2025-11-11T17:21:50.558861Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519294999342563:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:50.513607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519289577268506:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.610241Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.514266Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409057:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.518606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519293328409053:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(154922786ns) {Configure:104(154916533ns) {ProcessConfigured:135(25210306ns) {StartInitialHandshake:157(25206836ns) {DropHandshakes:469(13960991ns) {DropIncomingHandshake:441(1429ns) DropOutgoingHandshake:456(677ns)} GenerateSessionVirtualId:988(731ns) PrepareNewSessionHandshake:476(3915ns) {SwitchToState:178(517ns)}}}}}
2025-11-11T17:21:50.518775Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409058:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.518993Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409064:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.529852Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409063:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.599680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519293328409054:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(102290996ns) {Configure:104(102287030ns) {ProcessConfigured:135(11517532ns) {StartInitialHandshake:157(9122254ns) {DropHandshakes:469(3740ns) {DropIncomingHandshake:441(367ns) DropOutgoingHandshake:456(582ns)} GenerateSessionVirtualId:988(1444ns) PrepareNewSessionHandshake:476(3794ns) {SwitchToState:178(1612ns)}}}}}
2025-11-11T17:21:50.557997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519293872235903:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16726
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.523174Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871463:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.551371Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519292910720454:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:50.642159Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409056:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.642195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519293328409056:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.643118Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519293328409056:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:50.559379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519294999342547:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.553616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519292910720444:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.553688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:50.569870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519292910720444:2108] [node 5] ICP22 created new session: [4:7571519292910720466:2108]
2025-11-11T17:21:50.578098Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:50.580929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519292910720466:2108] [node 5] ICS09 handshake done sender: [4:7571519292910720454:2116] self: [4:7571519292910720453:0] peer: [5:7571519294999342565:0] socket: 158 qp: -1
2025-11-11T17:21:50.581990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519292910720466:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:50.591515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519288615753084:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519292910720455:2117] held: no
2025-11-11T17:21:50.597706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:50.598710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720467:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
2025-11-11T17:21:50.641369Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519292910720462:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.641565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519288615753083:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.559178Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519293872235898:2133] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:50.525868Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871462:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.645255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519293328409049:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.628149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519286980855892:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519291275823264:2115] held: no
2025-11-11T17:21:50.559435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.561180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519289577268506:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.650142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.525897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519292004871462:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.559476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519294999342547:2108] [node 4] ICP22 created new session: [5:7571519294999342567:2108]
2025-11-11T17:21:50.561224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:50.650210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519293328409049:2107] [node 4] ICP22 created new session: [1:7571519293328409078:2107]
2025-11-11T17:21:50.628854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823268:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.651166Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519292910720467:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.530524Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871465:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.559522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519294999342567:2108] [node 4] ICS09 handshake done sender: [5:7571519294999342563:2116] self: [5:7571519294999342565:0] peer: [4:7571519292910720453:0] socket: 159 qp: -1
2025-11-11T17:21:50.650310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519293328409078:2107] [node 4] ICS09 handshake done sender: [1:7571519293328409056:2111] self: [1:7571519293328409055:0] peer: [4:7571519292910720471:0] socket: 168 qp: -1
2025-11-11T17:21:50.561266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519289577268506:2049] [node 2] ICP22 created new session: [3:7571519293872235904:2049]
2025-11-11T17:21:50.530558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519292004871465:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.559558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519294999342567:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:50.629043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823267:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.677724Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519292910720452:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.650356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519293328409078:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:50.561312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519293872235904:2049] [node 2] ICS09 handshake done sender: [3:7571519293872235898:2133] self: [3:7571519293872235902:0] peer: [2:7571519292004871461:0] socket: 154 qp: -1
2025-11-11T17:21:50.559594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.534060Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871460:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.561341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519293872235904:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:50.629186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823269:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.561379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:50.569948Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235886:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.569987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519293872235886:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.581469Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519293872235886:2128] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:50.604206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519293872235884:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.604274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:50.604340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519293872235884:2114] [node 6] ICP22 created new session: [3:7571519293872235908:2114]
2025-11-11T17:21:50.604415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519293872235908:2114] [node 6] ICS09 handshake done sender: [3:7571519293872235886:2128] self: [3:7571519293872235885:0] peer: [6:7571519291275823265:0] socket: 160 qp: -1
2025-11-11T17:21:50.604455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519293872235908:2114] [node 6] ICS10 traffic start
2025-11-11T17:21:50.604492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:50.622869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519293872235911:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16726
2025-11-11T17:21:50.641770Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235911:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.650935Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235901:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.655943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519289577268505:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.702588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519293872235914:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16726
2025-11-11T17:21:50.730997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519293872235915:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16726
2025-11-11T17:21:50.735808Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235900:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.735848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519293872235900:2134] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.736502Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235894:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.736522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519293872235894:2132] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.737940Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519293872235901:2135] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:50.650406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:50.677772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519292910720452:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.741763Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519293872235900:2134] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:50.744785Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519293872235894:2132] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:50.534101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519292004871460:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.538712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871471:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.538906Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871470:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.541416Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519292004871462:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:50.541838Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519292004871465:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:50.542221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519287709904095:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.542288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:50.542333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519287709904095:2049] [node 3] ICP22 created new session: [2:7571519292004871480:2049]
2025-11-11T17:21:50.542377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519292004871480:2049] [node 3] ICS09 handshake done sender: [2:7571519292004871462:2114] self: [2:7571519292004871461:0] peer: [3:7571519293872235902:0] socket: 151 qp: -1
2025-11-11T17:21:50.542409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519292004871480:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:50.542447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:50.543404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519292004871457:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.543449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:50.543492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519292004871457:2107] [node 4] ICP22 created new session: [2:7571519292004871482:2107]
2025-11-11T17:21:50.543551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519292004871482:2107] [node 4] ICS09 handshake done sender: [2:7571519292004871465:2116] self: [2:7571519292004871464:0] peer: [4:7571519292910720463:0] socket: 152 qp: -1
2025-11-11T17:21:50.543583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519292004871482:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:50.543611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:50.601718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519287709904094:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20198679ns) {Configure:104(20191296ns) {ProcessConfigured:135(61313ns) {StartInitialHandshake:157(58170ns) {DropHandshakes:469(2521ns) {DropIncomingHandshake:441(760ns) DropOutgoingHandshake:456(570ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:476(3984ns) {SwitchToState:178(394ns)}}}}}
2025-11-11T17:21:50.611419Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519292004871460:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:50.647896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519292004871456:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.647961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:50.653167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519292004871456:2108] [node 5] ICP22 created new session: [2:7571519292004871489:2108]
2025-11-11T17:21:50.658165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519292004871489:2108] [node 5] ICS09 handshake done sender: [2:7571519292004871460:2113] self: [2:7571519292004871459:0] peer: [5:7571519294999342564:0] socket: 150 qp: -1
2025-11-11T17:21:50.658214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519292004871489:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:50.670554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:50.691973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519292004871456:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(43252998ns) {HandleHandshakeStatus:329(43249878ns) {DropIncomingHandshake:441(14838ns) DropOutgoingHandshake:456(846ns) SwitchToState:178(405ns) ProcessPendingSessionEvents:495(10620146ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10562505ns max# 10158844ns}}}
2025-11-11T17:21:50.692114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871474:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.700736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519287709904094:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.734688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519292004871492:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3136
2025-11-11T17:21:50.739565Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519292004871474:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:50.739791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519287709904094:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.739882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:50.739932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519287709904094:2048] [node 1] ICP22 created new session: [2:7571519292004871493:2048]
2025-11-11T17:21:50.739980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519292004871493:2048] [node 1] ICS09 handshake done sender: [2:7571519292004871474:2120] self: [2:7571519292004871491:0] peer: [1:7571519293328409059:0] socket: 171 qp: -1
2025-11-11T17:21:50.740014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519292004871493:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:50.740052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.622012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342570:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.667528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519293328409076:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28236
2025-11-11T17:21:50.638189Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519291275823267:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.688816Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:50.766623Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519292004871472:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.766665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519292004871472:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.767615Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519292004871472:2119] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:50.767871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519292004871455:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.767973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:50.768018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519292004871455:2109] [node 6] ICP22 created new session: [2:7571519292004871496:2109]
2025-11-11T17:21:50.768082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519292004871496:2109] [node 6] ICS09 handshake done sender: [2:7571519292004871472:2119] self: [2:7571519292004871469:0] peer: [6:7571519291275823278:0] socket: 185 qp: -1
2025-11-11T17:21:50.768117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519292004871496:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:50.768175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:50.647850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342571:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
2025-11-11T17:21:50.762138Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:50.774257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519289577268505:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.639498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519291275823246:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.718106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519288615753084:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(26262849ns) {HandleHandshakeStatus:395(26256286ns) {LogHandshakeFail:472(15624ns) {UpdateErrorStateLog:544(5540ns)} DropOutgoingHandshake:456(554ns) ProcessPendingSessionEvents:495(26177933ns) {unordered DropSessionEvent:512 num# 100 sum# 26131079ns max# 7912417ns}}}
2025-11-11T17:21:50.726779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519288615753083:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519292910720462:2119] held: no
2025-11-11T17:21:50.727441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519288615753085:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.727944Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519292910720455:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:50.745104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519288615753084:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.745199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:50.745245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519288615753084:2049] [node 2] ICP22 created new session: [4:7571519292910720476:2049]
2025-11-11T17:21:50.745293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519292910720476:2049] [node 2] ICS09 handshake done sender: [4:7571519292910720455:2117] self: [4:7571519292910720463:0] peer: [2:7571519292004871464:0] socket: 155 qp: -1
2025-11-11T17:21:50.745324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519292910720476:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:50.745354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:50.749956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720472:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
2025-11-11T17:21:50.753469Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519292910720452:2115] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:50.754211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519292910720445:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.775955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:50.680222Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409062:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.657754Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519294999342552:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.647917Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519291275823268:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.774404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.774451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519289577268505:2048] [node 1] ICP22 created new session: [3:7571519293872235918:2048]
2025-11-11T17:21:50.774494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519293872235918:2048] [node 1] ICS09 handshake done sender: [3:7571519293872235901:2135] self: [3:7571519293872235913:0] peer: [1:7571519293328409061:0] socket: 173 qp: -1
2025-11-11T17:21:50.774532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519293872235918:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:50.774571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:50.775194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7571519289577268505:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:50.775525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519293872235883:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.775556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:50.775608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519293872235883:2108] [node 5] ICP22 created new session: [3:7571519293872235920:2108]
2025-11-11T17:21:50.775650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519293872235920:2108] [node 5] ICS09 handshake done sender: [3:7571519293872235900:2134] self: [3:7571519293872235899:0] peer: [5:7571519294999342577:0] socket: 190 qp: -1
2025-11-11T17:21:50.775687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519293872235920:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:50.775717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:50.799179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519293872235882:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.799244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:50.806361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519293872235882:2107] [node 4] ICP22 created new session: [3:7571519293872235922:2107]
2025-11-11T17:21:50.680260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519293328409062:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.657789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519294999342552:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.656362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519291275823259:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.789007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519292910720445:2109] [node 6] ICP22 created new session: [4:7571519292910720478:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.701522Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519293328409062:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:50.666759Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519294999342561:2114] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:50.797741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519292910720478:2109] [node 6] ICS09 handshake done sender: [4:7571519292910720452:2115] self: [4:7571519292910720451:0] peer: [6:7571519291275823272:0] socket: 182 qp: -1
2025-11-11T17:21:50.685873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519290704375189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.672508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823266:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.813056Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519293872235915:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.709910Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409060:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.718278Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519291275823264:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:50.797795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519292910720478:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:50.685937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:50.813349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519293872235922:2107] [node 4] ICS09 handshake done sender: [3:7571519293872235894:2132] self: [3:7571519293872235893:0] peer: [4:7571519292910720474:0] socket: 181 qp: -1
2025-11-11T17:21:50.797842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.742030Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519291275823269:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.685994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519290704375189:2049] [node 2] ICP22 created new session: [5:7571519294999342574:2049]
2025-11-11T17:21:50.805967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519292910720444:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(251484486ns) {HandleHandshakeStatus:329(251479262ns) {DropIncomingHandshake:441(14682ns) DropOutgoingHandshake:456(569ns) SwitchToState:178(473ns) ProcessPendingSessionEvents:495(185867806ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 175159127ns max# 17189362ns}}}
2025-11-11T17:21:50.806074Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519292910720462:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:50.813471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720475:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
2025-11-11T17:21:50.832329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519292910720445:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(44325028ns) {HandleHandshakeStatus:329(44322908ns) {DropIncomingHandshake:441(672ns) DropOutgoingHandshake:456(1147ns) SwitchToState:178(679ns) ProcessPendingSessionEvents:495(517563ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 453452ns max# 44297ns}}}
2025-11-11T17:21:50.834196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519288615753083:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.834250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:50.834312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519288615753083:2048] [node 1] ICP22 created new session: [4:7571519292910720480:2048]
2025-11-11T17:21:50.834386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519292910720480:2048] [node 1] ICS09 handshake done sender: [4:7571519292910720462:2119] self: [4:7571519292910720471:0] peer: [1:7571519293328409055:0] socket: 172 qp: -1
2025-11-11T17:21:50.834430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519292910720480:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:50.834469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:50.837373Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519292910720467:2121] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:50.859753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519288615753085:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.859860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:50.859911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519288615753085:2050] [node 3] ICP22 created new session: [4:7571519292910720482:2050]
2025-11-11T17:21:50.859958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519292910720482:2050] [node 3] ICS09 handshake done sender: [4:7571519292910720467:2121] self: [4:7571519292910720474:0] peer: [3:7571519293872235893:0] socket: 148 qp: -1
2025-11-11T17:21:50.860002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519292910720482:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:50.860033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:50.883110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519292910720484:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6271
2025-11-11T17:21:50.883199Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:50.709945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519293328409060:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.830822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519293872235922:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:50.751404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519286980855892:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.836573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:50.858768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519293872235883:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(82466180ns) {HandleHandshakeStatus:329(82463254ns) {DropIncomingHandshake:441(290ns) DropOutgoingHandshake:456(370ns) SwitchToState:178(276ns) ProcessPendingSessionEvents:495(82193770ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 82084836ns max# 21411442ns}}}
2025-11-11T17:21:50.721826Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519293328409060:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.686039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519294999342574:2049] [node 2] ICS09 handshake done sender: [5:7571519294999342561:2114] self: [5:7571519294999342564:0] peer: [2:7571519292004871459:0] socket: 153 qp: -1
2025-11-11T17:21:50.722026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519289033441693:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.722128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:50.722173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519289033441693:2049] [node 3] ICP22 created new session: [1:7571519293328409083:2049]
2025-11-11T17:21:50.722228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519293328409083:2049] [node 3] ICS09 handshake done sender: [1:7571519293328409062:2115] self: [1:7571519293328409061:0] peer: [3:7571519293872235913:0] socket: 170 qp: -1
2025-11-11T17:21:50.722269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519293328409083:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:50.722307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:50.746016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519289033441692:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.746057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:50.746089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519289033441692:2048] [node 2] ICP22 created new session: [1:7571519293328409086:2048]
2025-11-11T17:21:50.749775Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409076:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.769779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519293328409086:2048] [node 2] ICS09 handshake done sender: [1:7571519293328409060:2114] self: [1:7571519293328409059:0] peer: [2:7571519292004871491:0] socket: 169 qp: -1
2025-11-11T17:21:50.776146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519293328409086:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:50.792579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:50.806768Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409074:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.806806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519293328409074:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.811884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519289033441692:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(65025457ns) {HandleHandshakeStatus:329(65007044ns) {DropIncomingHandshake:441(241ns) DropOutgoingHandshake:456(422ns) SwitchToState:178(416ns) ProcessPendingSessionEvents:495(8487075ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8422586ns max# 8005451ns}}}
2025-11-11T17:21:50.812034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519289033441692:2048] [node 2] ICP18 handshake (actor [1:7571519293328409076:2120]) is too old
2025-11-11T17:21:50.836054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519293328409074:2119] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:50.853293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519293328409054:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.867687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:50.877799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519293328409054:2108] [node 5] ICP22 created new session: [1:7571519293328409089:2108]
2025-11-11T17:21:50.890562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519293328409089:2108] [node 5] ICS09 handshake done sender: [1:7571519293328409074:2119] self: [1:7571519293328409073:0] peer: [5:7571519294999342583:0] socket: 201 qp: -1
2025-11-11T17:21:50.890622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519293328409089:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:50.893728Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519293328409069:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.893773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519293328409069:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:50.899714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:50.751486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:50.686084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519294999342574:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:50.905091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519292004871455:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(31039609ns) {HandleHandshakeStatus:329(31031768ns) {DropIncomingHandshake:441(11813ns) DropOutgoingHandshake:456(612ns) SwitchToState:178(861ns) ProcessPendingSessionEvents:495(30626432ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30550227ns max# 14490266ns}}}
2025-11-11T17:21:50.686123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:50.689836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342572:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
2025-11-11T17:21:50.701380Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519294999342552:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:50.717678Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519294999342571:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.751538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519286980855892:2050] [node 3] ICP22 created new session: [6:7571519291275823275:2050]
2025-11-11T17:21:50.718457Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519294999342572:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.719417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519290704375190:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.751581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519291275823275:2050] [node 3] ICS09 handshake done sender: [6:7571519291275823264:2115] self: [6:7571519291275823265:0] peer: [3:7571519293872235885:0] socket: 163 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:50.726231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519294999342548:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.726301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:50.729034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342580:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
2025-11-11T17:21:50.731179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519294999342548:2109] [node 6] ICP22 created new session: [5:7571519294999342579:2109]
2025-11-11T17:21:50.742296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519294999342579:2109] [node 6] ICS09 handshake done sender: [5:7571519294999342552:2111] self: [5:7571519294999342551:0] peer: [6:7571519291275823271:0] socket: 164 qp: -1
2025-11-11T17:21:50.742347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519294999342579:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:50.742423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:50.743834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519294999342548:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(16819386ns) {HandleHandshakeStatus:329(16817573ns) {DropIncomingHandshake:441(19016ns) DropOutgoingHandshake:456(619ns) SwitchToState:178(546ns) ProcessPendingSessionEvents:495(502174ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 448081ns max# 41646ns}}}
2025-11-11T17:21:50.765761Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519294999342580:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.772684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342582:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
2025-11-11T17:21:50.778895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519290704375188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.799004Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519294999342571:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:50.908549Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519293328409069:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:50.808903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519290704375190:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.751619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519291275823275:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:50.909658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519293872235882:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(108810333ns) {HandleHandshakeStatus:329(108807229ns) {DropIncomingHandshake:441(18500ns) DropOutgoingHandshake:456(632ns) SwitchToState:178(514ns) ProcessPendingSessionEvents:495(64920992ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 64792830ns max# 13219226ns}}}
2025-11-11T17:21:50.829302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519294999342584:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4606
2025-11-11T17:21:50.916653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519293328409054:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(62600195ns) {HandleHandshakeStatus:329(62596139ns) {DropIncomingHandshake:441(23064ns) DropOutgoingHandshake:456(389ns) SwitchToState:178(4177ns) ProcessPendingSessionEvents:495(10018291ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9912646ns max# 4217394ns}}}
2025-11-11T17:21:50.751651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:50.752461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823274:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.843249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:50.917292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519293328409053:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:50.845897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519290704375190:2050] [node 3] ICP22 created new session: [5:7571519294999342585:2050]
2025-11-11T17:21:50.850028Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519294999342580:2121] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:50.850444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519290704375188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.853980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:50.864318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519294999342585:2050] [node 3] ICS09 handshake done sender: [5:7571519294999342571:2119] self: [5:7571519294999342577:0] peer: [3:7571519293872235899:0] socket: 191 qp: -1
2025-11-11T17:21:50.867601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519290704375188:2048] [node 1] ICP22 created new session: [5:7571519294999342586:2048]
2025-11-11T17:21:50.874804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519294999342585:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:50.888558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:50.890560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519294999342586:2048] [node 1] ICS09 handshake done sender: [5:7571519294999342580:2121] self: [5:7571519294999342583:0] peer: [1:7571519293328409073:0] socket: 192 qp: -1
2025-11-11T17:21:50.890637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519294999342586:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:50.896439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519290704375190:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(86683424ns) {HandleHandshakeStatus:329(86679640ns) {DropIncomingHandshake:441(817ns) DropOutgoingHandshake:456(4222012ns) SwitchToState:178(1242ns) ProcessPendingSessionEvents:495(533102ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 470027ns max# 68830ns}}}
2025-11-11T17:21:50.898052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:50.763136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823273:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.918482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:50.763951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519286980855891:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.940426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519293872235883:2108] [node 5] ICP18 handshake (actor [3:7571519293872235915:2139]) is too old
2025-11-11T17:21:50.938934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519290704375188:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(61691028ns) {HandleHandshakeStatus:329(61687686ns) {DropIncomingHandshake:441(1435ns) DropOutgoingHandshake:456(377977ns) SwitchToState:178(849ns) ProcessPendingSessionEvents:495(5479609ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5430119ns max# 3038158ns}}}
2025-11-11T17:21:50.764021Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519291275823267:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:50.764401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519291275823246:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.764503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:50.764555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519291275823246:2104] [node 5] ICP22 created new session: [6:7571519291275823279:2104]
2025-11-11T17:21:50.764603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519291275823279:2104] [node 5] ICS09 handshake done sender: [6:7571519291275823267:2117] self: [6:7571519291275823271:0] peer: [5:7571519294999342551:0] socket: 144 qp: -1
2025-11-11T17:21:50.764640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519291275823279:2104] [node 5] ICS10 traffic start
2025-11-11T17:21:50.764686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:50.765365Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:50.808908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823277:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.815945Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519291275823273:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:50.816046Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:50.862444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519286980855891:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519291275823269:2119] held: no
2025-11-11T17:21:50.863545Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519291275823268:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:50.871776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519286980855890:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:50.918850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519293328409053:2109] [node 6] ICP22 created new session: [1:7571519293328409092:2109]
2025-11-11T17:21:50.872015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519291275823259:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.872115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:50.872166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519291275823259:2107] [node 4] ICP22 created new session: [6:7571519291275823284:2107]
2025-11-11T17:21:50.872244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519291275823284:2107] [node 4] ICS09 handshake done sender: [6:7571519291275823268:2118] self: [6:7571519291275823272:0] peer: [4:7571519292910720451:0] socket: 187 qp: -1
2025-11-11T17:21:50.872281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519291275823284:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:50.872318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:50.904515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823281:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.907015Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519291275823269:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:50.910106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519286980855891:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.910177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:50.910221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519286980855891:2049] [node 2] ICP22 created new session: [6:7571519291275823287:2049]
2025-11-11T17:21:50.910272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519291275823287:2049] [node 2] ICS09 handshake done sender: [6:7571519291275823269:2119] self: [6:7571519291275823278:0] peer: [2:7571519292004871469:0] socket: 189 qp: -1
2025-11-11T17:21:50.910303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519291275823287:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:50.910339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:50.915253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519291275823286:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-11T17:21:50.915763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519291275823246:2104] [node 5] ICP35 event processing took too much time PendingConnection:278(121340178ns) {HandleHandshakeStatus:329(121338357ns) {DropIncomingHandshake:441(496ns) DropOutgoingHandshake:456(4932ns) SwitchToState:178(170ns) ProcessPendingSessionEvents:495(120869254ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 118532450ns max# 10821159ns}}}
2025-11-11T17:21:50.916176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519291275823246:2104] [node 5] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:50.918509Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519291275823273:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:50.919646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519286980855890:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:50.924314Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:50.933616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:50.935128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519286980855890:2048] [node 1] ICP22 created new session: [6:7571519291275823289:2048]
2025-11-11T17:21:50.935897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519291275823289:2048] [node 1] ICS09 handshake done sender: [6:7571519291275823273:2120] self: [6:7571519291275823283:0] peer: [1:7571519293328409068:0] socket: 195 qp: -1
2025-11-11T17:21:50.935940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519291275823289:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:50.935978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:50.937338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519286980855890:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16990369ns) {HandleHandshakeStatus:329(16988378ns) {DropIncomingHandshake:441(521ns) DropOutgoingHandshake:456(3444ns) SwitchToState:178(614ns) ProcessPendingSessionEvents:495(498132ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 448786ns max# 47851ns}}}
2025-11-11T17:21:50.937456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519286980855890:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:50.922113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519293328409092:2109] [node 6] ICS09 handshake done sender: [1:7571519293328409069:2118] self: [1:7571519293328409068:0] peer: [6:7571519291275823283:0] socket: 194 qp: -1
2025-11-11T17:21:50.922155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519293328409092:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:50.931506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:51.013415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:51.013671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519292004871493:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-11T17:21:51.013710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519287709904094:2048] [node 1] ICP30 unregister session Session# [2:7571519292004871493:2048] VirtualId# [2:7571519292004871491:0]
2025-11-11T17:21:51.013742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519292004871493:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.013834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:51.013880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:51.017747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:51.017699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:51.018070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519292910720480:2048] [node 1] ICS01 socket: 172 reason# EndOfStream
2025-11-11T17:21:51.018104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519288615753083:2048] [node 1] ICP30 unregister session Session# [4:7571519292910720480:2048] VirtualId# [4:7571519292910720471:0]
2025-11-11T17:21:51.018139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519292910720480:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.018207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:51.018244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:51.018574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519291275823289:2048] [node 1] ICS07 socket disconnect 195 reason# EndOfStream
2025-11-11T17:21:51.018607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519291275823289:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.018655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:51.018688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519291275823289:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:51.017957Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:51.018124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519294999342586:2048] [node 1] ICS07 socket disconnect 192 reason# EndOfStream
2025-11-11T17:21:51.017071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:51.017333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [3:7571519293872235918:2048] [node 1] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-11T17:21:51.017369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519293872235918:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.017496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:51.017548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [3:7571519293872235918:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:51.018928Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address
2025-11-11T17:21:51.019198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519289577268505:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:51.019236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519293872235918:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:51.019263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519289577268505:2048] [node 1] ICP30 unregister session Session# [3:7571519293872235918:2048] VirtualId# [3:7571519293872235913:0]
2025-11-11T17:21:51.019384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:51.019419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7571519289577268505:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:51.019433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:51.018146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519294999342586:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.018210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:51.018248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519294999342586:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:51.019417Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address
2025-11-11T17:21:51.025689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519290704375188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:51.026393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519294999342586:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:51.026428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519290704375188:2048] [node 1] ICP30 unregister session Session# [5:7571519294999342586:2048] VirtualId# [5:7571519294999342583:0]
2025-11-11T17:21:51.026507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:51.026533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519290704375188:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:51.026546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:51.020614Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address
2025-11-11T17:21:51.025764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519286980855890:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:51.025820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519291275823289:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:51.025864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519286980855890:2048] [node 1] ICP30 unregister session Session# [6:7571519291275823289:2048] VirtualId# [6:7571519291275823283:0]
2025-11-11T17:21:51.026005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:51.026045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519286980855890:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:51.026058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19566) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:51.032593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7571519289577268505:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:51.038290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519286980855890:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:51.038171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519290704375188:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:51.090188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:51.090443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519293872235904:2049] [node 2] ICS01 socket: 154 reason# EndOfStream
2025-11-11T17:21:51.090483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519289577268506:2049] [node 2] ICP30 unregister session Session# [3:7571519293872235904:2049] VirtualId# [3:7571519293872235902:0]
2025-11-11T17:21:51.090520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519293872235904:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.090635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:51.090685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:51.095987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:51.096214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519292910720476:2049] [node 2] ICS01 socket: 155 reason# EndOfStream
2025-11-11T17:21:51.096260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519288615753084:2049] [node 2] ICP30 unregister session Session# [4:7571519292910720476:2049] VirtualId# [4:7571519292910720463:0]
2025-11-11T17:21:51.096295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519292910720476:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.096383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:51.096428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:51.096947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:51.097122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519291275823287:2049] [node 2] ICS01 socket: 189 reason# EndOfStream
2025-11-11T17:21:51.096528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:51.097158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519286980855891:2049] [node 2] ICP30 unregister session Session# [6:7571519291275823287:2049] VirtualId# [6:7571519291275823278:0]
2025-11-11T17:21:51.097183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519291275823287:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.096698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519294999342574:2049] [node 2] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:21:51.097225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:51.097257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:51.096734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519290704375189:2049] [node 2] ICP30 unregister session Session# [5:7571519294999342574:2049] VirtualId# [5:7571519294999342564:0]
2025-11-11T17:21:51.096770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519294999342574:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.096827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:51.096864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:51.189822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:51.190070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519292910720482:2050] [node 3] ICS01 socket: 148 reason# EndOfStream
2025-11-11T17:21:51.190112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519288615753085:2050] [node 3] ICP30 unregister session Session# [4:7571519292910720482:2050] VirtualId# [4:7571519292910720474:0]
2025-11-11T17:21:51.190144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519292910720482:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.190249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:51.190298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:51.190813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:51.191031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519291275823275:2050] [node 3] ICS01 socket: 163 reason# EndOfStream
2025-11-11T17:21:51.191064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519286980855892:2050] [node 3] ICP30 unregister session Session# [6:7571519291275823275:2050] VirtualId# [6:7571519291275823265:0]
2025-11-11T17:21:51.191093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519291275823275:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.191140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:51.191170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:51.190391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:51.190590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519294999342585:2050] [node 3] ICS01 socket: 191 reason# EndOfStream
2025-11-11T17:21:51.190621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519290704375190:2050] [node 3] ICP30 unregister session Session# [5:7571519294999342585:2050] VirtualId# [5:7571519294999342577:0]
2025-11-11T17:21:51.190655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519294999342585:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.190701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:51.190735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:51.295015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:51.295286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519294999342567:2108] [node 4] ICS01 socket: 159 reason# EndOfStream
2025-11-11T17:21:51.295326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519294999342547:2108] [node 4] ICP30 unregister session Session# [5:7571519294999342567:2108] VirtualId# [5:7571519294999342565:0]
2025-11-11T17:21:51.295379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519294999342567:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.295483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:51.295539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:51.295654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:51.295827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519291275823284:2107] [node 4] ICS01 socket: 187 reason# EndOfStream
2025-11-11T17:21:51.295864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519291275823259:2107] [node 4] ICP30 unregister session Session# [6:7571519291275823284:2107] VirtualId# [6:7571519291275823272:0]
2025-11-11T17:21:51.295894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519291275823284:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.295952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:51.295989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:51.371068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:51.371340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519291275823279:2104] [node 5] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-11T17:21:51.371387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519291275823279:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:51.371509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:51.371551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519291275823279:2104] [node 5] ICS15 start handshake
2025-11-11T17:21:51.376186Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65041) Couldn't connect to any resolved address
2025-11-11T17:21:51.376505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519291275823246:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65041) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:51.376568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519291275823279:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:51.376609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519291275823246:2104] [node 5] ICP30 unregister session Session# [6:7571519291275823279:2104] VirtualId# [6:7571519291275823271:0]
2025-11-11T17:21:51.376707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
2025-11-11T17:21:51.376742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519291275823246:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65041) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:51.376758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65041) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:51.493613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519297547954405:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:51.493740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519297547954406:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:51.517223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519299052325391:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:51.517267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519299052325392:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:51.543491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519295875027455:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:51.543546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519295875027456:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:51.568588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519299135902305:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:51.568645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519299135902306:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:51.568686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519299135902307:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:51.591012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519299580091646:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:51.591051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519299580091647:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:51.591076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519299580091648:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:51.614760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519296324806913:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:51.614782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519296324806914:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:51.619111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519296324806915:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:51.962135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519296324806955:2088] [node 4] ICP01 ready to work
2025-11-11T17:21:51.993658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519299580091707:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.099061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774274:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.090113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519301842921766:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:52.086442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519299580091707:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(40358957ns) {Configure:104(40349936ns) {ProcessConfigured:135(71799ns) {StartInitialHandshake:157(69755ns) {DropHandshakes:469(2152ns) {DropIncomingHandshake:441(465ns) DropOutgoingHandshake:456(519ns)} GenerateSessionVirtualId:988(730ns) PrepareNewSessionHandshake:476(1515ns) {SwitchToState:178(516ns)}}}}}
2025-11-11T17:21:52.128566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519303430869663:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:52.130473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519300619774278:2101] [node 5] ICP01 ready to work
2025-11-11T17:21:52.130373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519301842921767:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:52.130148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519303347292750:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:52.158833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519299135902306:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(29990686ns) {Configure:104(29986528ns) {ProcessConfigured:135(50333ns) {StartInitialHandshake:157(48160ns) {DropHandshakes:469(2423ns) {DropIncomingHandshake:441(450ns) DropOutgoingHandshake:456(250ns)} GenerateSessionVirtualId:988(1493ns) PrepareNewSessionHandshake:476(1839ns) {SwitchToState:178(480ns)}}}}}
2025-11-11T17:21:52.159136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519303430869667:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.146985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519299052325391:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(40529474ns) {Configure:104(40518318ns) {ProcessConfigured:135(84754ns) {StartInitialHandshake:157(81941ns) {DropHandshakes:469(3861ns) {DropIncomingHandshake:441(906ns) DropOutgoingHandshake:456(719ns)} GenerateSessionVirtualId:988(996ns) PrepareNewSessionHandshake:476(2544ns) {SwitchToState:178(659ns)}}}}}
2025-11-11T17:21:52.140255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519301842921768:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:52.183764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519296324806915:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(221697294ns) {Configure:104(221693212ns) {ProcessConfigured:135(85134ns) {StartInitialHandshake:157(82255ns) {DropHandshakes:469(3623ns) {DropIncomingHandshake:441(673ns) DropOutgoingHandshake:456(466ns)} GenerateSessionVirtualId:988(5758ns) PrepareNewSessionHandshake:476(1573ns) {SwitchToState:178(712ns)}}}}}
2025-11-11T17:21:52.205391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519300169994814:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:52.210712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774285:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.178050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519303347292751:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:52.219365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519303347292758:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:52.205645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519300169994816:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:52.242975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519303875059009:2110] [node 4] ICP01 ready to work
2025-11-11T17:21:52.271097Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519300619774274:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.211019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869670:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-11-11T17:21:52.205436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519300169994815:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:52.244643Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519303430869670:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.291868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519301842921769:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54842
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.350458Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519303347292755:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.350503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519303347292755:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.351718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519303347292763:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29710
2025-11-11T17:21:52.352042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519303347292765:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29710
2025-11-11T17:21:52.352710Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519303347292755:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:52.353261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519303347292750:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.315693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869676:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.360549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869677:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-11-11T17:21:52.309213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519299580091648:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(142270479ns) {Configure:104(142263075ns) {ProcessConfigured:135(61415ns) {StartInitialHandshake:157(59358ns) {DropHandshakes:469(2982ns) {DropIncomingHandshake:441(532ns) DropOutgoingHandshake:456(889ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:476(1435ns) {SwitchToState:178(406ns)}}}}}
2025-11-11T17:21:52.317673Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519300619774285:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.301954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519297547954405:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(322277932ns) {Configure:104(322262827ns) {ProcessConfigured:135(13437603ns) {StartInitialHandshake:157(13435277ns) {DropHandshakes:469(2761ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(1051ns)} GenerateSessionVirtualId:988(595ns) PrepareNewSessionHandshake:476(5960ns) {SwitchToState:178(2629ns)}}}}}
2025-11-11T17:21:52.353324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:52.321934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059016:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.353385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519303347292750:2109] [node 6] ICP22 created new session: [2:7571519303347292767:2109]
2025-11-11T17:21:52.358091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519303347292767:2109] [node 6] ICS09 handshake done sender: [2:7571519303347292755:2111] self: [2:7571519303347292754:0] peer: [6:7571519300619774287:0] socket: 151 qp: -1
2025-11-11T17:21:52.358147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519303347292767:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:52.358192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:52.359320Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519303347292763:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.359813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519303347292751:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(138299889ns) {Configure:104(138294400ns) {ProcessConfigured:135(17623883ns) {StartInitialHandshake:157(17620533ns) {DropHandshakes:469(2872ns) {DropIncomingHandshake:441(975ns) DropOutgoingHandshake:456(413ns)} GenerateSessionVirtualId:988(861ns) PrepareNewSessionHandshake:476(3007ns) {SwitchToState:178(988ns)}}}}}
2025-11-11T17:21:52.302505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519301842921771:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54842
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.326162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519296324806914:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.352478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774288:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.353369Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519300619774285:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:52.353633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519296324806914:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.361829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:52.373364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519296324806914:2049] [node 2] ICP22 created new session: [6:7571519300619774289:2049]
2025-11-11T17:21:52.402031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519300619774289:2049] [node 2] ICS09 handshake done sender: [6:7571519300619774285:2115] self: [6:7571519300619774287:0] peer: [2:7571519303347292754:0] socket: 152 qp: -1
2025-11-11T17:21:52.402088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519300619774289:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:52.402151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:52.403949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519296324806914:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(49319396ns) {HandleHandshakeStatus:329(49317193ns) {DropIncomingHandshake:441(1032ns) DropOutgoingHandshake:456(952393ns) SwitchToState:178(5704ns) ProcessPendingSessionEvents:495(571999ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 513656ns max# 72695ns}}}
2025-11-11T17:21:52.377615Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519303430869676:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.411427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519303347292758:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(98381235ns) {Configure:104(98372759ns) {ProcessConfigured:135(100964ns) {StartInitialHandshake:157(98182ns) {DropHandshakes:469(2821ns) {DropIncomingHandshake:441(640ns) DropOutgoingHandshake:456(541ns)} GenerateSessionVirtualId:988(1010ns) PrepareNewSessionHandshake:476(4847ns) {SwitchToState:178(605ns)}}}}}
2025-11-11T17:21:52.390028Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519303875059016:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.318400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519301842921770:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54842
2025-11-11T17:21:52.382209Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921769:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.382320Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921770:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.386037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519301842921766:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(133976682ns) {Configure:104(133972755ns) {ProcessConfigured:135(22131160ns) {StartInitialHandshake:157(22126428ns) {DropHandshakes:469(2426ns) {DropIncomingHandshake:441(582ns) DropOutgoingHandshake:456(409ns)} GenerateSessionVirtualId:988(952ns) PrepareNewSessionHandshake:476(6390ns) {SwitchToState:178(2363ns)}}}}}
2025-11-11T17:21:52.412336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519297547954406:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(193501964ns) {Configure:104(193495920ns) {ProcessConfigured:135(26992558ns) {StartInitialHandshake:157(26988210ns) {DropHandshakes:469(6445476ns) {DropIncomingHandshake:441(581ns) DropOutgoingHandshake:456(1762ns)} GenerateSessionVirtualId:988(2234ns) PrepareNewSessionHandshake:476(1693ns) {SwitchToState:178(209ns)}}}}}
2025-11-11T17:21:52.412671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921777:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.412694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519301842921777:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.414170Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921771:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.420282Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921776:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.420334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519301842921776:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.422239Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519301842921777:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:52.438747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519300169994816:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(207062359ns) {Configure:104(207055087ns) {ProcessConfigured:135(65272ns) {StartInitialHandshake:157(61039ns) {DropHandshakes:469(3242ns) {DropIncomingHandshake:441(814ns) DropOutgoingHandshake:456(579ns)} GenerateSessionVirtualId:988(892ns) PrepareNewSessionHandshake:476(2167ns) {SwitchToState:178(769ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.443944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519300169994817:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-11T17:21:52.419076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519303875059009:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(140305512ns) {Configure:104(140295418ns) {ProcessConfigured:135(107006ns) {StartInitialHandshake:157(102879ns) {DropHandshakes:469(3551ns) {DropIncomingHandshake:441(807ns) DropOutgoingHandshake:456(677ns)} GenerateSessionVirtualId:988(1591ns) PrepareNewSessionHandshake:476(2598ns) {SwitchToState:178(928ns)}}}}}
2025-11-11T17:21:52.390391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519303430869667:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(228803371ns) {Configure:104(228799102ns) {ProcessConfigured:135(16887770ns) {StartInitialHandshake:157(16882571ns) {DropHandshakes:469(4219ns) {DropIncomingHandshake:441(1350ns) DropOutgoingHandshake:456(659ns)} GenerateSessionVirtualId:988(1352ns) PrepareNewSessionHandshake:476(7651ns) {SwitchToState:178(1988ns)}}}}}
2025-11-11T17:21:52.421736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519299580091646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(252550275ns) {Configure:104(252536669ns) {ProcessConfigured:135(36035837ns) {StartInitialHandshake:157(36030130ns) {DropHandshakes:469(2214ns) {DropIncomingHandshake:441(438ns) DropOutgoingHandshake:456(446ns)} GenerateSessionVirtualId:988(1143ns) PrepareNewSessionHandshake:476(3642ns) {SwitchToState:178(1326ns)}}}}}
2025-11-11T17:21:52.411618Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519303347292765:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.458620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519300169994818:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-11T17:21:52.411898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519299052325391:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.409851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519299135902305:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.471120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519300619774278:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(87577284ns) {Configure:104(87570099ns) {ProcessConfigured:135(89398ns) {StartInitialHandshake:157(86397ns) {DropHandshakes:469(2404ns) {DropIncomingHandshake:441(525ns) DropOutgoingHandshake:456(399ns)} GenerateSessionVirtualId:988(830ns) PrepareNewSessionHandshake:476(2008ns) {SwitchToState:178(528ns)}}}}}
2025-11-11T17:21:52.422530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519299135902305:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32074809ns) {IncomingHandshake:261(32068581ns) {DropIncomingHandshake:441(2153ns) PrepareNewSessionHandshake:476(3601ns) {SwitchToState:178(2797ns)} GenerateSessionVirtualId:988(1097ns) IssueIncomingHandshakeReply:193(8021651ns) {SwitchToState:178(1439ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.490914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519300169994814:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(259203995ns) {Configure:104(259198252ns) {ProcessConfigured:135(72290ns) {StartInitialHandshake:157(69982ns) {DropHandshakes:469(3288ns) {DropIncomingHandshake:441(583ns) DropOutgoingHandshake:456(482ns)} GenerateSessionVirtualId:988(1025ns) PrepareNewSessionHandshake:476(1561ns) {SwitchToState:178(419ns)}}}}}
2025-11-11T17:21:52.417642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519303347292762:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29710
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.498092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519297547954405:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.499510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059021:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.471697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519300619774278:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.543784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774295:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.543960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774296:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.544641Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519300619774274:2109] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:52.442052Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519303347292762:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.491192Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994817:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.472564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519303430869663:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(243985944ns) {Configure:104(234334375ns) {ProcessConfigured:135(66677ns) {StartInitialHandshake:157(64167ns) {DropHandshakes:469(1966ns) {DropIncomingHandshake:441(428ns) DropOutgoingHandshake:456(490ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:476(1781ns) {SwitchToState:178(503ns)}}}}}
2025-11-11T17:21:52.516988Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519301842921776:2116] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:52.544933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519300619774278:2101] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.544992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:52.545036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519300619774278:2101] [node 5] ICP22 created new session: [6:7571519300619774297:2101]
2025-11-11T17:21:52.545076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519300619774297:2101] [node 5] ICS09 handshake done sender: [6:7571519300619774274:2109] self: [6:7571519300619774293:0] peer: [5:7571519303875059005:0] socket: 147 qp: -1
2025-11-11T17:21:52.545114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519300619774297:2101] [node 5] ICS10 traffic start
2025-11-11T17:21:52.545150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.512835Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519303875059006:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.512890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519303875059006:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.522102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059022:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.513876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519303347292775:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29710
2025-11-11T17:21:52.536977Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519303875059006:2109] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:52.575991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519299580091646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.539754Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519303347292765:2115] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:52.491287Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994818:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.542027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519299052325391:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.542095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:52.499880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519300169994819:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-11T17:21:52.501656Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519303430869677:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.522955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.516862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519300169994824:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-11T17:21:52.583321Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519300619774295:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.542156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519299052325391:2048] [node 1] ICP22 created new session: [2:7571519303347292777:2048]
2025-11-11T17:21:52.525855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519299135902306:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.531578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519297547954405:2048] [node 2] ICP22 created new session: [1:7571519301842921791:2048]
2025-11-11T17:21:52.525745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519295875027455:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(31571352ns) {Configure:104(31565219ns) {ProcessConfigured:135(14136982ns) {StartInitialHandshake:157(14133238ns) {DropHandshakes:469(3700ns) {DropIncomingHandshake:441(706ns) DropOutgoingHandshake:456(524ns)} GenerateSessionVirtualId:988(1086ns) PrepareNewSessionHandshake:476(3255ns) {SwitchToState:178(367ns)}}}}}
2025-11-11T17:21:52.526587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-11-11T17:21:52.526736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869686:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-11-11T17:21:52.534937Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519303430869676:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:52.537251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519299135902305:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.549995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:52.550065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519299135902305:2048] [node 1] ICP22 created new session: [4:7571519303430869687:2048]
2025-11-11T17:21:52.550121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519303430869687:2048] [node 1] ICS09 handshake done sender: [4:7571519303430869676:2114] self: [4:7571519303430869681:0] peer: [1:7571519301842921775:0] socket: 171 qp: -1
2025-11-11T17:21:52.550159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519303430869687:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:52.550198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:52.553014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519303430869686:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.553153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519299135902307:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.578566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869690:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-11-11T17:21:52.596502Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519303430869686:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:52.596864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869692:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-11-11T17:21:52.598550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519299135902307:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.598616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:52.598673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519299135902307:2050] [node 3] ICP22 created new session: [4:7571519303430869693:2050]
2025-11-11T17:21:52.598720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519303430869693:2050] [node 3] ICS09 handshake done sender: [4:7571519303430869686:2119] self: [4:7571519303430869689:0] peer: [3:7571519300169994822:0] socket: 189 qp: -1
2025-11-11T17:21:52.598759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519303430869693:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:52.598804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:52.618745Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519303430869677:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:52.619030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519299135902306:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.619118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:52.619173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519299135902306:2049] [node 2] ICP22 created new session: [4:7571519303430869696:2049]
2025-11-11T17:21:52.619226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519303430869696:2049] [node 2] ICS09 handshake done sender: [4:7571519303430869677:2115] self: [4:7571519303430869684:0] peer: [2:7571519303347292764:0] socket: 176 qp: -1
2025-11-11T17:21:52.619262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519303430869696:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:52.619303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.542215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519303347292777:2048] [node 1] ICS09 handshake done sender: [2:7571519303347292765:2115] self: [2:7571519303347292774:0] peer: [1:7571519301842921774:0] socket: 172 qp: -1
2025-11-11T17:21:52.613178Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 6] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveExRequestHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.651707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519303430869698:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-11-11T17:21:52.665746Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519303430869698:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.613503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519299580091707:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:52.609710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519296324806913:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.562137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519301842921768:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.631296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059026:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.530017Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994819:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.542259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519303347292777:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:52.632887Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519303875059016:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:52.562200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.721741Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519303430869679:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.553488Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994823:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.621980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774300:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.542301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:52.633089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519299580091646:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.562254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519301842921768:2107] [node 4] ICP22 created new session: [1:7571519301842921793:2107]
2025-11-11T17:21:52.721797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519303430869679:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.553515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519300169994823:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.558335Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:52.669864Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519300619774300:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.769701Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519303430869683:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.769770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519303430869683:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.562327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519301842921793:2107] [node 4] ICS09 handshake done sender: [1:7571519301842921776:2116] self: [1:7571519301842921775:0] peer: [4:7571519303430869681:0] socket: 167 qp: -1
2025-11-11T17:21:52.777698Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519303430869679:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:52.633170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:52.698962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774301:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.576043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519295875027456:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.562369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519301842921793:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:52.562402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:52.569813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519301842921791:2048] [node 2] ICS09 handshake done sender: [1:7571519301842921777:2117] self: [1:7571519301842921774:0] peer: [2:7571519303347292774:0] socket: 168 qp: -1
2025-11-11T17:21:52.577848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519301842921791:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:52.578908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:52.582922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519301842921768:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(20011569ns) {HandleHandshakeStatus:329(20007633ns) {DropIncomingHandshake:441(13940ns) DropOutgoingHandshake:456(483ns) SwitchToState:178(344ns) ProcessPendingSessionEvents:495(19528088ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19463389ns max# 18955430ns}}}
2025-11-11T17:21:52.589837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519301842921792:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54842
2025-11-11T17:21:52.589982Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921773:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.590006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519301842921773:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.601959Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519301842921773:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:52.646075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519301842921767:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.646127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:52.646181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519301842921767:2108] [node 5] ICP22 created new session: [1:7571519301842921798:2108]
2025-11-11T17:21:52.646231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519301842921798:2108] [node 5] ICS09 handshake done sender: [1:7571519301842921773:2115] self: [1:7571519301842921772:0] peer: [5:7571519303875059024:0] socket: 166 qp: -1
2025-11-11T17:21:52.646266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519301842921798:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:52.646303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:52.666020Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921782:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.666052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519301842921782:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.676133Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519301842921782:2118] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:52.712100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519297547954405:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(133912717ns) {HandleHandshakeStatus:329(133908712ns) {DropIncomingHandshake:441(16951ns) DropOutgoingHandshake:456(1015ns) SwitchToState:178(1019ns) ProcessPendingSessionEvents:495(17973565ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17917085ns max# 17471095ns}}}
2025-11-11T17:21:52.721679Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921792:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.725640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519301842921766:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.765992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:52.776803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519301842921766:2109] [node 6] ICP22 created new session: [1:7571519301842921802:2109]
2025-11-11T17:21:52.788344Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519301842921786:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.788369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519301842921786:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.585725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519303347292769:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.633214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519299580091646:2048] [node 1] ICP22 created new session: [5:7571519303875059027:2048]
2025-11-11T17:21:52.577352Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994824:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.578993Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519300169994823:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:52.586064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519295875027456:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(38904208ns) {IncomingHandshake:261(16358347ns) {DropIncomingHandshake:441(592ns) PrepareNewSessionHandshake:476(1827ns) {SwitchToState:178(586ns)} GenerateSessionVirtualId:988(1343ns) IssueIncomingHandshakeReply:193(63513ns) {SwitchToState:178(452ns)}}}
2025-11-11T17:21:52.678157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519300169994839:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-11T17:21:52.678308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519300169994840:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-11T17:21:52.690526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519300169994815:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.690600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:52.690650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519300169994815:2107] [node 4] ICP22 created new session: [3:7571519300169994841:2107]
2025-11-11T17:21:52.690704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519300169994841:2107] [node 4] ICS09 handshake done sender: [3:7571519300169994823:2114] self: [3:7571519300169994822:0] peer: [4:7571519303430869689:0] socket: 187 qp: -1
2025-11-11T17:21:52.690750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519300169994841:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:52.690787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:52.698409Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519300169994819:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:52.714161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519295875027456:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.714260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:52.714308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519295875027456:2049] [node 2] ICP22 created new session: [3:7571519300169994844:2049]
2025-11-11T17:21:52.714357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519300169994844:2049] [node 2] ICS09 handshake done sender: [3:7571519300169994819:2112] self: [3:7571519300169994836:0] peer: [2:7571519303347292759:0] socket: 163 qp: -1
2025-11-11T17:21:52.714391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519300169994844:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:52.714425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:52.745900Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994839:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.746946Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994821:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.746973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519300169994821:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.770916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519300169994821:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:52.777693Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519300169994828:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.777734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519300169994828:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.785842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519295875027455:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.713776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519296324806955:2088] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.585772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519303347292769:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.785997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519295875027455:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19819362ns) {IncomingHandshake:261(19817299ns) {DropIncomingHandshake:441(310ns) PrepareNewSessionHandshake:476(4959ns) {SwitchToState:178(1061ns)} GenerateSessionVirtualId:988(549ns) IssueIncomingHandshakeReply:193(49622ns) {SwitchToState:178(123ns)}}}
2025-11-11T17:21:52.633262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519303875059027:2048] [node 1] ICS09 handshake done sender: [5:7571519303875059016:2113] self: [5:7571519303875059024:0] peer: [1:7571519301842921772:0] socket: 169 qp: -1
2025-11-11T17:21:52.802735Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519301842921786:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:52.805974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519303430869667:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.601767Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519303347292769:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.724687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774303:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.811550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519301842921802:2109] [node 6] ICS09 handshake done sender: [1:7571519301842921782:2118] self: [1:7571519301842921781:0] peer: [6:7571519300619774299:0] socket: 183 qp: -1
2025-11-11T17:21:52.633302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519303875059027:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:52.806099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:52.794599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519300169994816:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.609672Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519303347292760:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.818569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519301842921802:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:52.821664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:52.846982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519297547954406:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.847101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:52.847181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519297547954406:2049] [node 3] ICP22 created new session: [1:7571519301842921805:2049]
2025-11-11T17:21:52.847240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519301842921805:2049] [node 3] ICS09 handshake done sender: [1:7571519301842921786:2119] self: [1:7571519301842921785:0] peer: [3:7571519300169994847:0] socket: 154 qp: -1
2025-11-11T17:21:52.847279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519301842921805:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:52.847337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:52.729981Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519300619774303:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.806153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519303430869667:2109] [node 6] ICP22 created new session: [4:7571519303430869702:2109]
2025-11-11T17:21:52.633334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:52.609745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519303347292760:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.799315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:52.730114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519296324806915:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.634092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519299580091707:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.802466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519300169994816:2108] [node 5] ICP22 created new session: [3:7571519300169994848:2108]
2025-11-11T17:21:52.806198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519303430869702:2109] [node 6] ICS09 handshake done sender: [4:7571519303430869679:2116] self: [4:7571519303430869678:0] peer: [6:7571519300619774302:0] socket: 196 qp: -1
2025-11-11T17:21:52.746045Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519300619774295:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:52.758963Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:52.776504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519296324806913:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.785694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:52.794570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519296324806913:2048] [node 1] ICP22 created new session: [6:7571519300619774305:2048]
2025-11-11T17:21:52.807991Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:52.811706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519300619774305:2048] [node 1] ICS09 handshake done sender: [6:7571519300619774295:2118] self: [6:7571519300619774299:0] peer: [1:7571519301842921781:0] socket: 188 qp: -1
2025-11-11T17:21:52.817736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519300619774305:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:52.820228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519296324806955:2088] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519300619774300:2120] held: no
2025-11-11T17:21:52.821332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:52.837876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774307:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.842080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519300619774278:2101] [node 5] ICP35 event processing took too much time PendingConnection:278(111334340ns) {HandleHandshakeStatus:329(111331895ns) {DropIncomingHandshake:441(779ns) DropOutgoingHandshake:456(4655ns) SwitchToState:178(479ns) ProcessPendingSessionEvents:495(110981992ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 110865352ns max# 19545150ns}}}
2025-11-11T17:21:52.867195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519296324806913:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(106377874ns) {HandleHandshakeStatus:329(106374202ns) {DropIncomingHandshake:441(2772ns) DropOutgoingHandshake:456(7032ns) SwitchToState:178(602ns) ProcessPendingSessionEvents:495(16518694ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16451709ns max# 16039708ns}}}
2025-11-11T17:21:52.867328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519296324806913:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:52.873696Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519300619774300:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:52.873747Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:52.880134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519296324806915:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519300619774303:2122] held: no
2025-11-11T17:21:52.657694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519303347292751:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.893940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519296324806955:2088] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.894050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:52.894103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519296324806955:2088] [node 4] ICP22 created new session: [6:7571519300619774309:2088]
2025-11-11T17:21:52.894146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519300619774309:2088] [node 4] ICS09 handshake done sender: [6:7571519300619774300:2120] self: [6:7571519300619774302:0] peer: [4:7571519303430869678:0] socket: 197 qp: -1
2025-11-11T17:21:52.894192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519300619774309:2088] [node 4] ICS10 traffic start
2025-11-11T17:21:52.894227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:52.895360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519300619774308:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61052
2025-11-11T17:21:52.829357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519300169994848:2108] [node 5] ICS09 handshake done sender: [3:7571519300169994821:2113] self: [3:7571519300169994820:0] peer: [5:7571519303875059033:0] socket: 184 qp: -1
2025-11-11T17:21:52.906203Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519300619774303:2122] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:52.906883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519296324806915:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.906963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:52.907018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519296324806915:2050] [node 3] ICP22 created new session: [6:7571519300619774311:2050]
2025-11-11T17:21:52.907066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519300619774311:2050] [node 3] ICS09 handshake done sender: [6:7571519300619774303:2122] self: [6:7571519300619774304:0] peer: [3:7571519300169994827:0] socket: 207 qp: -1
2025-11-11T17:21:52.907106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519300619774311:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:52.907145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:52.806237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519303430869702:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:52.634157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:52.667973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:52.806275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:52.815661Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519303430869683:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:52.829753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519303430869663:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.829825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:52.829880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519303430869663:2108] [node 5] ICP22 created new session: [4:7571519303430869704:2108]
2025-11-11T17:21:52.846779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519303430869704:2108] [node 5] ICS09 handshake done sender: [4:7571519303430869683:2117] self: [4:7571519303430869682:0] peer: [5:7571519303875059034:0] socket: 155 qp: -1
2025-11-11T17:21:52.850158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519303430869704:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:52.850205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:52.681958Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519303347292760:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:52.685776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519303347292751:2107] [node 4] ICP22 created new session: [2:7571519303347292782:2107]
2025-11-11T17:21:52.709901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519303347292782:2107] [node 4] ICS09 handshake done sender: [2:7571519303347292769:2116] self: [2:7571519303347292764:0] peer: [4:7571519303430869684:0] socket: 175 qp: -1
2025-11-11T17:21:52.725029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519303347292782:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:52.727015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519299052325392:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.727073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:52.727153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519299052325392:2049] [node 3] ICP22 created new session: [2:7571519303347292783:2049]
2025-11-11T17:21:52.727242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519303347292783:2049] [node 3] ICS09 handshake done sender: [2:7571519303347292760:2112] self: [2:7571519303347292759:0] peer: [3:7571519300169994836:0] socket: 160 qp: -1
2025-11-11T17:21:52.727284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519303347292783:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:52.727323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:52.745756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:52.762887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519303347292772:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.762911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519303347292772:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:52.764354Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519303347292772:2117] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:52.786447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519303347292758:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.786531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:52.786583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519303347292758:2108] [node 5] ICP22 created new session: [2:7571519303347292787:2108]
2025-11-11T17:21:52.786631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519303347292787:2108] [node 5] ICS09 handshake done sender: [2:7571519303347292772:2117] self: [2:7571519303347292771:0] peer: [5:7571519303875059035:0] socket: 149 qp: -1
2025-11-11T17:21:52.786665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519303347292787:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:52.786729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:52.791203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519303347292751:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(147266201ns) {HandleHandshakeStatus:329(132724581ns) {DropIncomingHandshake:441(19622ns) DropOutgoingHandshake:456(744ns) SwitchToState:178(705ns) ProcessPendingSessionEvents:495(12647413ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12583981ns max# 12180292ns}}}
2025-11-11T17:21:52.816603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519299052325391:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(208632615ns) {HandleHandshakeStatus:329(208614595ns) {DropIncomingHandshake:441(1485ns) DropOutgoingHandshake:456(7964ns) SwitchToState:178(821ns) ProcessPendingSessionEvents:495(208147976ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 200925092ns max# 20819365ns}}}
2025-11-11T17:21:52.817058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519299052325391:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:52.634204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519299580091707:2108] [node 6] ICP22 created new session: [5:7571519303875059029:2108]
2025-11-11T17:21:52.634246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519303875059029:2108] [node 6] ICS09 handshake done sender: [5:7571519303875059006:2109] self: [5:7571519303875059005:0] peer: [6:7571519300619774293:0] socket: 146 qp: -1
2025-11-11T17:21:52.634278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519303875059029:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:52.634335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:52.697922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059032:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.717867Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519303875059022:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.718482Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519303875059032:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.725466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519299580091648:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.725573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519303875059009:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.750124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.762411Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519303875059025:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:52.927661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519303430869663:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(96971844ns) {HandleHandshakeStatus:329(96967483ns) {DropIncomingHandshake:441(16399ns) DropOutgoingHandshake:456(610ns) SwitchToState:178(997ns) ProcessPendingSessionEvents:495(51098946ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 51024940ns max# 36821953ns}}}
2025-11-11T17:21:52.830252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519300169994848:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:52.762560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519299580091647:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:52.764124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059036:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.764878Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519303875059025:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:52.765141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519299580091647:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.765206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:52.765249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519299580091647:2049] [node 2] ICP22 created new session: [5:7571519303875059037:2049]
2025-11-11T17:21:52.765290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519303875059037:2049] [node 2] ICS09 handshake done sender: [5:7571519303875059025:2118] self: [5:7571519303875059035:0] peer: [2:7571519303347292771:0] socket: 198 qp: -1
2025-11-11T17:21:52.765327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519303875059037:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:52.765389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:52.770717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059039:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.780690Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519303875059022:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:52.781055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519299580091648:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.781107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:52.781158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519299580091648:2050] [node 3] ICP22 created new session: [5:7571519303875059041:2050]
2025-11-11T17:21:52.781202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519303875059041:2050] [node 3] ICS09 handshake done sender: [5:7571519303875059022:2117] self: [5:7571519303875059033:0] peer: [3:7571519300169994820:0] socket: 185 qp: -1
2025-11-11T17:21:52.781253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519303875059041:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:52.781289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:52.825698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519299580091646:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(189178850ns) {HandleHandshakeStatus:329(189172180ns) {DropIncomingHandshake:441(1074ns) DropOutgoingHandshake:456(5466ns) SwitchToState:178(352ns) ProcessPendingSessionEvents:495(188727650ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 183654933ns max# 22266025ns}}}
2025-11-11T17:21:52.844945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519303875059040:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54534
2025-11-11T17:21:52.856080Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:52.869096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519303875059009:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519303875059032:2120] held: no
2025-11-11T17:21:52.881353Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519303875059032:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:52.905826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519303875059009:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.905952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:52.913938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519303875059009:2110] [node 4] ICP22 created new session: [5:7571519303875059043:2110]
2025-11-11T17:21:52.921533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519303875059043:2110] [node 4] ICS09 handshake done sender: [5:7571519303875059032:2120] self: [5:7571519303875059034:0] peer: [4:7571519303430869682:0] socket: 153 qp: -1
2025-11-11T17:21:52.928589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519299580091648:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(146610831ns) {HandleHandshakeStatus:329(146608827ns) {DropIncomingHandshake:441(1039ns) DropOutgoingHandshake:456(5160ns) SwitchToState:178(857ns) ProcessPendingSessionEvents:495(146237211ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 146081417ns max# 16124319ns}}}
2025-11-11T17:21:52.929262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519303875059043:2110] [node 4] ICS10 traffic start
2025-11-11T17:21:52.929322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:52.929499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519303875059009:2110] [node 4] ICP35 event processing took too much time PendingConnection:278(30595211ns) {HandleHandshakeStatus:329(30593028ns) {DropIncomingHandshake:441(2507ns) DropOutgoingHandshake:456(819ns) SwitchToState:178(1203ns) ProcessPendingSessionEvents:495(960ns)}}
2025-11-11T17:21:52.830294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:52.837655Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519300169994828:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:52.859695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519300169994814:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:52.881246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:52.884957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519300169994814:2109] [node 6] ICP22 created new session: [3:7571519300169994852:2109]
2025-11-11T17:21:52.897820Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:52.901668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519300169994852:2109] [node 6] ICS09 handshake done sender: [3:7571519300169994828:2117] self: [3:7571519300169994827:0] peer: [6:7571519300619774304:0] socket: 206 qp: -1
2025-11-11T17:21:52.916326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519300169994852:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:52.916398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:52.918123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519300169994814:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(57498172ns) {HandleHandshakeStatus:329(57494615ns) {DropIncomingHandshake:441(16065ns) DropOutgoingHandshake:456(638ns) SwitchToState:178(774ns) ProcessPendingSessionEvents:495(604558ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 546416ns max# 49863ns}}}
2025-11-11T17:21:52.918221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519295875027455:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519300169994839:2119] held: no
2025-11-11T17:21:52.928472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519300169994851:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-11T17:21:52.930129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519300169994816:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(134720486ns) {HandleHandshakeStatus:329(134715630ns) {DropIncomingHandshake:441(16418ns) DropOutgoingHandshake:456(698ns) SwitchToState:178(1707ns) ProcessPendingSessionEvents:495(59057987ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 58980137ns max# 13630316ns}}}
2025-11-11T17:21:52.931068Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519300169994839:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:52.931334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519295875027455:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:52.932377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:52.932433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519295875027455:2048] [node 1] ICP22 created new session: [3:7571519300169994854:2048]
2025-11-11T17:21:52.932484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519300169994854:2048] [node 1] ICS09 handshake done sender: [3:7571519300169994839:2119] self: [3:7571519300169994847:0] peer: [1:7571519301842921785:0] socket: 202 qp: -1
2025-11-11T17:21:52.932518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519300169994854:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:52.932588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:52.938830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519299580091647:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(172996589ns) {HandleHandshakeStatus:329(172993822ns) {DropIncomingHandshake:441(1523ns) DropOutgoingHandshake:456(13944ns) SwitchToState:178(542ns) ProcessPendingSessionEvents:495(172621616ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 172476216ns max# 19798885ns}}}
2025-11-11T17:21:52.940010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519299135902307:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(333502493ns) {HandleHandshakeStatus:329(333498687ns) {DropIncomingHandshake:441(833ns) DropOutgoingHandshake:456(12013ns) SwitchToState:178(513ns) ProcessPendingSessionEvents:495(333075719ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 332856149ns max# 29126605ns}}}
2025-11-11T17:21:52.939888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519301842921766:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(217266396ns) {HandleHandshakeStatus:329(217263947ns) {DropIncomingHandshake:441(33104ns) DropOutgoingHandshake:456(646ns) SwitchToState:178(1046ns) ProcessPendingSessionEvents:495(108914308ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 108804146ns max# 19581855ns}}}
2025-11-11T17:21:52.982063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:52.982293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519303347292777:2048] [node 1] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-11T17:21:52.982329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519303347292777:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:52.982412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:52.982452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519303347292777:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:52.985733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:52.987931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519300169994854:2048] [node 1] ICS01 socket: 202 reason# EndOfStream
2025-11-11T17:21:52.987973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519295875027455:2048] [node 1] ICP30 unregister session Session# [3:7571519300169994854:2048] VirtualId# [3:7571519300169994847:0]
2025-11-11T17:21:52.988013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519300169994854:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:52.988088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:52.988138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:52.988812Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:21:52.989143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519299052325391:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:21:52.989182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519303347292777:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:52.991139Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address
2025-11-11T17:21:52.991405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519299052325391:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:52.991458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519303347292777:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:52.991497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519299052325391:2048] [node 1] ICP30 unregister session Session# [2:7571519303347292777:2048] VirtualId# [2:7571519303347292774:0]
2025-11-11T17:21:52.991600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:52.991633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571519299052325391:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:52.991649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:52.993435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:52.993685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519303430869687:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-11T17:21:52.993747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519299135902305:2048] [node 1] ICP30 unregister session Session# [4:7571519303430869687:2048] VirtualId# [4:7571519303430869681:0]
2025-11-11T17:21:52.993785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519303430869687:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:52.993865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:52.993906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:52.996387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:52.996604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519300619774305:2048] [node 1] ICS01 socket: 188 reason# EndOfStream
2025-11-11T17:21:52.996640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519296324806913:2048] [node 1] ICP30 unregister session Session# [6:7571519300619774305:2048] VirtualId# [6:7571519300619774299:0]
2025-11-11T17:21:52.996680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519300619774305:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:52.996747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:52.996784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:52.995283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:52.995565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519303875059027:2048] [node 1] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-11T17:21:52.995595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519303875059027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:52.995671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:52.995712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519303875059027:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:52.997036Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address
2025-11-11T17:21:53.001746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571519299052325391:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:53.009741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519299580091646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:53.010195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519303875059027:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:53.010237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519299580091646:2048] [node 1] ICP30 unregister session Session# [5:7571519303875059027:2048] VirtualId# [5:7571519303875059024:0]
2025-11-11T17:21:53.010357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:53.010393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519299580091646:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.010406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.021870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519299580091646:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:53.100266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:53.100497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519300169994844:2049] [node 2] ICS01 socket: 163 reason# EndOfStream
2025-11-11T17:21:53.100536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519295875027456:2049] [node 2] ICP30 unregister session Session# [3:7571519300169994844:2049] VirtualId# [3:7571519300169994836:0]
2025-11-11T17:21:53.100571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519300169994844:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.100660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:53.100705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:53.105786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:53.105773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:53.106010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519300619774289:2049] [node 2] ICS07 socket disconnect 152 reason# EndOfStream
2025-11-11T17:21:53.106010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519303875059037:2049] [node 2] ICS07 socket disconnect 198 reason# EndOfStream
2025-11-11T17:21:53.106045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519300619774289:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.106039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519303875059037:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.106133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:53.106124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:53.106166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519303875059037:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:53.106169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519300619774289:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:53.106468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:53.106654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519303430869696:2049] [node 2] ICS01 socket: 176 reason# EndOfStream
2025-11-11T17:21:53.106696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519299135902306:2049] [node 2] ICP30 unregister session Session# [4:7571519303430869696:2049] VirtualId# [4:7571519303430869684:0]
2025-11-11T17:21:53.106725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519303430869696:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.106796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:53.106841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:53.107861Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address
2025-11-11T17:21:53.107901Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address
2025-11-11T17:21:53.108220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519299580091647:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:53.108265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519303875059037:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:53.108303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519299580091647:2049] [node 2] ICP30 unregister session Session# [5:7571519303875059037:2049] VirtualId# [5:7571519303875059035:0]
2025-11-11T17:21:53.108386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:53.108416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519299580091647:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.108315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519296324806914:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:53.108430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.108368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519300619774289:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:53.108398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519296324806914:2049] [node 2] ICP30 unregister session Session# [6:7571519300619774289:2049] VirtualId# [6:7571519300619774287:0]
2025-11-11T17:21:53.108431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:53.108459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519296324806914:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.108473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.118783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519296324806914:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:53.125932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519299580091647:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:53.184212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:53.184580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519300619774311:2050] [node 3] ICS01 socket: 207 reason# EndOfStream
2025-11-11T17:21:53.184622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519296324806915:2050] [node 3] ICP30 unregister session Session# [6:7571519300619774311:2050] VirtualId# [6:7571519300619774304:0]
2025-11-11T17:21:53.184657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519300619774311:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.184745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:53.184791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:53.185864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:53.185830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:53.186110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519303875059041:2050] [node 3] ICS01 socket: 185 reason# EndOfStream
2025-11-11T17:21:53.186164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519299580091648:2050] [node 3] ICP30 unregister session Session# [5:7571519303875059041:2050] VirtualId# [5:7571519303875059033:0]
2025-11-11T17:21:53.186197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519303875059041:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.186306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:53.186968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:53.186017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519303430869693:2050] [node 3] ICS07 socket disconnect 189 reason# EndOfStream
2025-11-11T17:21:53.186056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519303430869693:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.186129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:53.186175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519303430869693:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:53.189246Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17109) Couldn't connect to any resolved address
2025-11-11T17:21:53.189654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519299135902307:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17109) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:53.189711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519303430869693:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:53.189752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519299135902307:2050] [node 3] ICP30 unregister session Session# [4:7571519303430869693:2050] VirtualId# [4:7571519303430869689:0]
2025-11-11T17:21:53.189843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:53.189885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519299135902307:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.189899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:53.202502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519299135902307:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:53.273810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:53.274044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519303875059043:2110] [node 4] ICS01 socket: 153 reason# EndOfStream
2025-11-11T17:21:53.274086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519303875059009:2110] [node 4] ICP30 unregister session Session# [5:7571519303875059043:2110] VirtualId# [5:7571519303875059034:0]
2025-11-11T17:21:53.274119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519303875059043:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.274206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:53.274258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:53.274361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:53.274546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519300619774309:2088] [node 4] ICS01 socket: 197 reason# EndOfStream
2025-11-11T17:21:53.274575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519296324806955:2088] [node 4] ICP30 unregister session Session# [6:7571519300619774309:2088] VirtualId# [6:7571519300619774302:0]
2025-11-11T17:21:53.274600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519300619774309:2088] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:53.274652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:53.274687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:53.514267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519305485317547:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:53.514310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519305485317548:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:53.593896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519305838396787:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:53.593937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519305838396788:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:53.617606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519307345513312:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:53.617649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519307345513313:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:53.665632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519306178516171:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:53.667894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519306178516172:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:53.667935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519306178516173:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:53.700009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519306771370101:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:53.700050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519306771370102:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:53.700078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519306771370103:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:53.724529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519304410154168:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:53.724568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519304410154169:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:53.724596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519304410154170:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:54.523802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519305485317548:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(174860322ns) {Configure:104(174855939ns) {ProcessConfigured:135(41457838ns) {StartInitialHandshake:157(41454150ns) {DropHandshakes:469(3070ns) {DropIncomingHandshake:441(559ns) DropOutgoingHandshake:456(579ns)} GenerateSessionVirtualId:988(1297ns) PrepareNewSessionHandshake:476(4305ns) {SwitchToState:178(1797ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.530906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519308705121487:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:54.522003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519311066337431:2073] [node 6] ICP01 ready to work
2025-11-11T17:21:54.542674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519310473483528:2101] [node 5] ICP01 ready to work
2025-11-11T17:21:54.557817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519310133364146:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:54.598627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519309780284937:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:54.598738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519309780284939:2114] [node 5] ICP01 ready to work
2025-11-11T17:21:54.542652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519311066337425:2070] [node 4] ICP01 ready to work
2025-11-11T17:21:54.558210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519311640480689:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:54.556876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519308705121488:2063] [node 4] ICP01 ready to work
2025-11-11T17:21:54.621680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519310133364147:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:54.631032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519305838396787:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(174687826ns) {Configure:104(146761318ns) {ProcessConfigured:135(34351ns) {StartInitialHandshake:157(33084ns) {DropHandshakes:469(989ns) {DropIncomingHandshake:441(321ns) DropOutgoingHandshake:456(99ns)} GenerateSessionVirtualId:988(416ns) PrepareNewSessionHandshake:476(654ns) {SwitchToState:178(358ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.579759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519306771370101:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(227528600ns) {Configure:104(227524124ns) {ProcessConfigured:135(13812651ns) {StartInitialHandshake:157(13809940ns) {DropHandshakes:469(2456ns) {DropIncomingHandshake:441(563ns) DropOutgoingHandshake:456(422ns)} GenerateSessionVirtualId:988(881ns) PrepareNewSessionHandshake:476(4301ns) {SwitchToState:178(2097ns)}}}}}
2025-11-11T17:21:54.545788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519310473483527:2100] [node 6] ICP01 ready to work
2025-11-11T17:21:54.584988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519311640480693:2107] [node 5] ICP01 ready to work
2025-11-11T17:21:54.562330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519308705121490:2067] [node 5] ICP01 ready to work
2025-11-11T17:21:54.618585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519309780284938:2118] [node 6] ICP01 ready to work
2025-11-11T17:21:54.642144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519310133364148:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:54.632390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519311640480694:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:54.627744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519304410154170:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(64877577ns) {Configure:104(64870986ns) {ProcessConfigured:135(61375ns) {StartInitialHandshake:157(59319ns) {DropHandshakes:469(2430ns) {DropIncomingHandshake:441(629ns) DropOutgoingHandshake:456(495ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:476(1873ns) {SwitchToState:178(1003ns)}}}}}
2025-11-11T17:21:54.639420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519309780284940:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52803
2025-11-11T17:21:54.657905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519305838396788:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(262713448ns) {Configure:104(262708855ns) {ProcessConfigured:135(53520ns) {StartInitialHandshake:157(51601ns) {DropHandshakes:469(4594ns) {DropIncomingHandshake:441(541ns) DropOutgoingHandshake:456(446ns)} GenerateSessionVirtualId:988(911ns) PrepareNewSessionHandshake:476(1360ns) {SwitchToState:178(591ns)}}}}}
2025-11-11T17:21:54.694294Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519310133364146:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(84084205ns) {Configure:104(84079016ns) {ProcessConfigured:135(97859ns) {StartInitialHandshake:157(94979ns) {DropHandshakes:469(2511ns) {DropIncomingHandshake:441(556ns) DropOutgoingHandshake:456(563ns)} GenerateSessionVirtualId:988(1079ns) PrepareNewSessionHandshake:476(1413ns) {SwitchToState:178(540ns)}}}}}
2025-11-11T17:21:54.639519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519309780284942:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52803
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.720161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519310133364158:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38695
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.729961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519311066337471:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:54.726167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519309780284937:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(108936401ns) {Configure:104(108931757ns) {ProcessConfigured:135(21594124ns) {StartInitialHandshake:157(21591329ns) {DropHandshakes:469(2518ns) {DropIncomingHandshake:441(736ns) DropOutgoingHandshake:456(423ns)} GenerateSessionVirtualId:988(701ns) PrepareNewSessionHandshake:476(4052ns) {SwitchToState:178(1742ns)}}}}}
2025-11-11T17:21:54.748911Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519308705121487:2070] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.753237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519311640480697:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34421
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.763422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519308705121490:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(118630128ns) {Configure:104(118623995ns) {ProcessConfigured:135(58203ns) {StartInitialHandshake:157(55502ns) {DropHandshakes:469(2627ns) {DropIncomingHandshake:441(587ns) DropOutgoingHandshake:456(443ns)} GenerateSessionVirtualId:988(1106ns) PrepareNewSessionHandshake:476(1664ns) {SwitchToState:178(457ns)}}}}}
2025-11-11T17:21:54.756029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519311640480695:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34421
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.780304Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519311066337433:2080] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.776997Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519309780284940:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.764253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519308705121535:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:54.781656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519308705121490:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.787017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519304410154168:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(168250261ns) {Configure:104(168244562ns) {ProcessConfigured:135(73323ns) {StartInitialHandshake:157(70542ns) {DropHandshakes:469(2425ns) {DropIncomingHandshake:441(542ns) DropOutgoingHandshake:456(376ns)} GenerateSessionVirtualId:988(711ns) PrepareNewSessionHandshake:476(1410ns) {SwitchToState:178(435ns)}}}}}
2025-11-11T17:21:54.794131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519304410154169:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(241136487ns) {Configure:104(241129415ns) {ProcessConfigured:135(48409ns) {StartInitialHandshake:157(45705ns) {DropHandshakes:469(2013ns) {DropIncomingHandshake:441(327ns) DropOutgoingHandshake:456(454ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:476(1631ns) {SwitchToState:178(635ns)}}}}}
2025-11-11T17:21:54.801516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519310473483533:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
2025-11-11T17:21:54.771240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519307345513312:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(59186693ns) {Configure:104(59183127ns) {ProcessConfigured:135(63591ns) {StartInitialHandshake:157(61227ns) {DropHandshakes:469(2682ns) {DropIncomingHandshake:441(660ns) DropOutgoingHandshake:456(480ns)} GenerateSessionVirtualId:988(710ns) PrepareNewSessionHandshake:476(4356ns) {SwitchToState:178(505ns)}}}}}
2025-11-11T17:21:54.832765Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519308705121535:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.780344Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519311066337433:2080] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:54.771460Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519311640480695:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.781645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519311640480697:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.832732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519311640480693:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(126472376ns) {Configure:104(126467491ns) {ProcessConfigured:135(43975560ns) {StartInitialHandshake:157(43973108ns) {DropHandshakes:469(799ns) {DropIncomingHandshake:441(265ns) DropOutgoingHandshake:456(113ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:476(6785ns) {SwitchToState:178(2468ns)}}}}}
2025-11-11T17:21:54.842512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519307345513313:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.788043Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519309780284942:2140] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.810600Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364158:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.870271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519307345513313:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(38622179ns) {IncomingHandshake:261(26535203ns) {DropIncomingHandshake:441(299ns) PrepareNewSessionHandshake:476(10211734ns) {SwitchToState:178(3907ns)} GenerateSessionVirtualId:988(1121ns) IssueIncomingHandshakeReply:193(16289827ns) {SwitchToState:178(2253ns)}}}
2025-11-11T17:21:54.791176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519311066337433:2080] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:54.810759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519305838396787:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.838093Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364150:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.838121Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519310133364150:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:54.839211Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519310133364150:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:54.839545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519305838396788:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:54.839595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:54.839638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519305838396788:2049] [node 3] ICP22 created new session: [2:7571519310133364164:2049]
2025-11-11T17:21:54.839683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519310133364164:2049] [node 3] ICS09 handshake done sender: [2:7571519310133364150:2110] self: [2:7571519310133364149:0] peer: [3:7571519311640480709:0] socket: 156 qp: -1
2025-11-11T17:21:54.839717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519310133364164:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:54.839769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:54.841946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519310133364147:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(169414080ns) {Configure:104(169405229ns) {ProcessConfigured:135(8794012ns) {StartInitialHandshake:157(8791606ns) {DropHandshakes:469(2238ns) {DropIncomingHandshake:441(342ns) DropOutgoingHandshake:456(637ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:476(3142ns) {SwitchToState:178(1043ns)}}}}}
2025-11-11T17:21:54.859107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519310133364168:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38695
2025-11-11T17:21:54.882257Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519311640480700:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.882292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519311640480700:2132] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.833644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519310473483534:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
2025-11-11T17:21:54.788785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519309780284948:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52803
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.870599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519304410154170:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(17282717ns)
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.877730Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519310473483534:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.794896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519311066337474:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:54.842266Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519309780284936:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.881853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519304410154170:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.884039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519306178516173:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(330092516ns) {Configure:104(330087404ns) {ProcessConfigured:135(6973144ns) {StartInitialHandshake:157(6970882ns) {DropHandshakes:469(2233ns) {DropIncomingHandshake:441(502ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(2274ns) PrepareNewSessionHandshake:476(2461ns) {SwitchToState:178(1201ns)}}}}}
2025-11-11T17:21:54.884235Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519310473483533:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.886808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519310473483532:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.890160Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519311640480700:2132] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:54.795044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519311066337475:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:54.842301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519309780284936:2138] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:54.890051Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519310133364158:2113] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:54.885983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519308705121546:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.894007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519311640480712:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34421
2025-11-11T17:21:54.890250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519305838396787:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.902126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519308705121544:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:54.937873Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519310473483538:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.803313Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519311066337471:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.869744Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519309780284936:2138] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.908300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519311640480710:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34421
2025-11-11T17:21:54.908393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519305485317547:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:54.923074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519310133364169:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38695
2025-11-11T17:21:54.909682Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:54.842214Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519311066337474:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.937908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519310473483538:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:54.909632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519311640480698:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34421
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.928246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:54.908461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:54.910832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519308705121490:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519308705121487:2070] held: no
2025-11-11T17:21:54.930446Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519308705121544:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.938505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519308705121543:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:54.938572Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519308705121535:2113] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:54.943608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519304410154169:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.961731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519304410154170:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.953436Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519310473483532:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.958134Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519311640480697:2130] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:54.974043Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519311640480698:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.974134Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519311640480712:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.974171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519311640480694:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:54.974236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:54.974290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519311640480694:2109] [node 6] ICP22 created new session: [3:7571519311640480713:2109]
2025-11-11T17:21:54.974331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519311640480713:2109] [node 6] ICS09 handshake done sender: [3:7571519311640480700:2132] self: [3:7571519311640480699:0] peer: [6:7571519308705121545:0] socket: 172 qp: -1
2025-11-11T17:21:54.974364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519311640480713:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:54.974435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:54.975515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519307345513313:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:54.975562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:54.975610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519307345513313:2049] [node 2] ICP22 created new session: [3:7571519311640480715:2049]
2025-11-11T17:21:54.975652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519311640480715:2049] [node 2] ICS09 handshake done sender: [3:7571519311640480697:2130] self: [3:7571519311640480709:0] peer: [2:7571519310133364149:0] socket: 166 qp: -1
2025-11-11T17:21:54.975687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519311640480715:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:54.975712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:54.991320Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519311640480691:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.991378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519311640480691:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.009493Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519311640480701:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.009525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519311640480701:2133] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.019439Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519311640480691:2128] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:55.009657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519307345513312:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:55.054573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519307345513312:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(32891497ns) {IncomingHandshake:261(32888994ns) {DropIncomingHandshake:441(479ns) PrepareNewSessionHandshake:476(1930ns) {SwitchToState:178(920ns)} GenerateSessionVirtualId:988(355ns) IssueIncomingHandshakeReply:193(32336896ns) {SwitchToState:178(4293ns)}}}
2025-11-11T17:21:55.057655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519311640480689:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(27608541ns) {IncomingHandshake:261(16887489ns) {DropIncomingHandshake:441(1177ns) PrepareNewSessionHandshake:476(3862ns) {SwitchToState:178(2986ns)} GenerateSessionVirtualId:988(746ns) IssueIncomingHandshakeReply:193(7293ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.842382Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519311066337475:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.930589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519305838396787:2048] [node 1] ICP22 created new session: [2:7571519310133364170:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:55.066269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519311640480689:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.066349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:55.078020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519311640480689:2108] [node 4] ICP22 created new session: [3:7571519315935448018:2108]
2025-11-11T17:21:54.923528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519305485317547:2048] [node 2] ICP22 created new session: [1:7571519309780284950:2048]
2025-11-11T17:21:54.964577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519308705121488:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(259788944ns) {Configure:104(259782502ns) {ProcessConfigured:135(10304211ns) {StartInitialHandshake:157(10301637ns) {DropHandshakes:469(1552ns) {DropIncomingHandshake:441(269ns) DropOutgoingHandshake:456(388ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:476(3215ns) {SwitchToState:178(1735ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.866407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519311066337431:2073] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.937784Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364156:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.927653Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519309780284948:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.953569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519306178516173:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.870353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:54.964645Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519308705121487:2070] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:54.957636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519310473483547:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
2025-11-11T17:21:55.106044Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519311640480701:2133] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:54.937821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519310133364156:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:54.930323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519309780284950:2048] [node 2] ICS09 handshake done sender: [1:7571519309780284936:2138] self: [1:7571519309780284935:0] peer: [2:7571519310133364161:0] socket: 165 qp: -1
2025-11-11T17:21:54.987308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519308705121552:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:54.993807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519308705121490:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.001925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:55.001995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519308705121490:2067] [node 5] ICP22 created new session: [6:7571519313000088849:2067]
2025-11-11T17:21:55.009653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:55.021719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519304410154170:2050] [node 3] ICP22 created new session: [6:7571519313000088850:2050]
2025-11-11T17:21:55.021935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519313000088849:2067] [node 5] ICS09 handshake done sender: [6:7571519308705121487:2070] self: [6:7571519308705121537:0] peer: [5:7571519311066337432:0] socket: 151 qp: -1
2025-11-11T17:21:55.021988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519313000088849:2067] [node 5] ICS10 traffic start
2025-11-11T17:21:55.042093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519308705121544:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:55.043558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:55.045803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519313000088850:2050] [node 3] ICS09 handshake done sender: [6:7571519308705121535:2113] self: [6:7571519308705121545:0] peer: [3:7571519311640480699:0] socket: 175 qp: -1
2025-11-11T17:21:55.054435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519313000088850:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:55.054489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:55.065759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519304410154169:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.096446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:55.110949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519313000088854:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:55.118216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519308705121490:2067] [node 5] ICP35 event processing took too much time PendingConnection:278(104178576ns) {HandleHandshakeStatus:329(104175990ns) {DropIncomingHandshake:441(864ns) DropOutgoingHandshake:456(426ns) SwitchToState:178(434ns) ProcessPendingSessionEvents:495(1432ns)}}
2025-11-11T17:21:55.120824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519304410154169:2049] [node 2] ICP22 created new session: [6:7571519313000088853:2049]
2025-11-11T17:21:55.135328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519304410154170:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(169908160ns) {HandleHandshakeStatus:329(169905258ns) {DropIncomingHandshake:441(955ns) DropOutgoingHandshake:456(5629ns) SwitchToState:178(3370ns) ProcessPendingSessionEvents:495(13571889ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13509127ns max# 13080158ns}}}
2025-11-11T17:21:55.137955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519313000088855:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:55.147763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519313000088853:2049] [node 2] ICS09 handshake done sender: [6:7571519308705121544:2117] self: [6:7571519308705121548:0] peer: [2:7571519310133364159:0] socket: 185 qp: -1
2025-11-11T17:21:55.147815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519313000088853:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:55.156903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:54.873829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519306771370102:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(375957637ns) {Configure:104(375949981ns) {ProcessConfigured:135(42934141ns) {StartInitialHandshake:157(42931626ns) {DropHandshakes:469(2641ns) {DropIncomingHandshake:441(398ns) DropOutgoingHandshake:456(460ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:476(3973ns) {SwitchToState:178(736ns)}}}}}
2025-11-11T17:21:54.941637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519310133364170:2048] [node 1] ICS09 handshake done sender: [2:7571519310133364158:2113] self: [2:7571519310133364161:0] peer: [1:7571519309780284935:0] socket: 167 qp: -1
2025-11-11T17:21:55.098073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519315935448018:2108] [node 4] ICS09 handshake done sender: [3:7571519311640480691:2128] self: [3:7571519311640480690:0] peer: [4:7571519310473483550:0] socket: 152 qp: -1
2025-11-11T17:21:54.965019Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519310473483547:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.934498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519309780284950:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:55.198086Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519313000088854:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.217993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519308705121488:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:55.110455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519315935448019:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34421
2025-11-11T17:21:54.886288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519306771370103:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(282192327ns) {Configure:104(282184158ns) {ProcessConfigured:135(20002917ns) {StartInitialHandshake:157(20000991ns) {DropHandshakes:469(12713782ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(2667ns)} GenerateSessionVirtualId:988(1995ns) PrepareNewSessionHandshake:476(2262ns) {SwitchToState:178(1300ns)}}}}}
2025-11-11T17:21:54.973716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519306178516172:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.942736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519309780284939:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(72753704ns) {Configure:104(72748187ns) {ProcessConfigured:135(55289ns) {StartInitialHandshake:157(52585ns) {DropHandshakes:469(3048ns) {DropIncomingHandshake:441(746ns) DropOutgoingHandshake:456(444ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:476(2087ns) {SwitchToState:178(504ns)}}}}}
2025-11-11T17:21:54.945089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:55.007077Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519309780284934:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.007105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519309780284934:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.011290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519309780284938:2118] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(101467854ns) {Configure:104(93467209ns) {ProcessConfigured:135(98534ns) {StartInitialHandshake:157(95769ns) {DropHandshakes:469(2347ns) {DropIncomingHandshake:441(563ns) DropOutgoingHandshake:456(419ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:476(1775ns) {SwitchToState:178(471ns)}}}}}
2025-11-11T17:21:55.037811Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519309780284945:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.037859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519309780284945:2141] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.061648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519314075252255:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52803
2025-11-11T17:21:55.064180Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519309780284945:2141] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:55.106473Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519314075252255:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.109824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519309780284937:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.109877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:55.116829Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519309780284934:2137] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:55.129813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519309780284937:2107] [node 4] ICP22 created new session: [1:7571519314075252259:2107]
2025-11-11T17:21:55.147507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519305485317548:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(23210890ns) {IncomingHandshake:261(23208685ns) {DropIncomingHandshake:441(519ns) PrepareNewSessionHandshake:476(1970ns) {SwitchToState:178(980ns)} GenerateSessionVirtualId:988(596ns) IssueIncomingHandshakeReply:193(9382ns)}}
2025-11-11T17:21:55.165859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519314075252259:2107] [node 4] ICS09 handshake done sender: [1:7571519309780284945:2141] self: [1:7571519309780284944:0] peer: [4:7571519310473483555:0] socket: 170 qp: -1
2025-11-11T17:21:55.165912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519305485317548:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.165990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:55.178836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519314075252259:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:55.181735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519305485317548:2049] [node 3] ICP22 created new session: [1:7571519314075252261:2049]
2025-11-11T17:21:55.193234Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519309780284953:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.193266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519309780284953:2143] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.217741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:55.218043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519314075252261:2049] [node 3] ICS09 handshake done sender: [1:7571519309780284934:2137] self: [1:7571519309780284933:0] peer: [3:7571519311640480718:0] socket: 158 qp: -1
2025-11-11T17:21:55.221887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519314075252261:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:54.946865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519310133364170:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:54.889769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519311066337431:2073] [node 6] ICP22 created new session: [5:7571519311066337478:2073]
2025-11-11T17:21:54.894473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519306771370103:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.894885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519311066337482:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:54.901734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519311066337478:2073] [node 6] ICS09 handshake done sender: [5:7571519311066337433:2080] self: [5:7571519311066337432:0] peer: [6:7571519308705121537:0] socket: 150 qp: -1
2025-11-11T17:21:54.902990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519311066337478:2073] [node 6] ICS10 traffic start
2025-11-11T17:21:54.903036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:54.906039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519306771370102:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.933959Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519311066337482:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.934130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519311066337425:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.967897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519311066337431:2073] [node 6] ICP35 event processing took too much time PendingConnection:278(100697310ns) {HandleHandshakeStatus:329(100694616ns) {DropIncomingHandshake:441(11504ns) DropOutgoingHandshake:456(619ns) SwitchToState:178(992ns) ProcessPendingSessionEvents:495(27434315ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27367588ns max# 26978219ns}}}
2025-11-11T17:21:55.036132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519315361304785:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:55.037093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519311066337482:2118] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:55.037364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519311066337425:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.037427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:55.037472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519311066337425:2070] [node 4] ICP22 created new session: [5:7571519315361304786:2070]
2025-11-11T17:21:55.037540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519315361304786:2070] [node 4] ICS09 handshake done sender: [5:7571519311066337482:2118] self: [5:7571519311066337486:0] peer: [4:7571519310473483537:0] socket: 181 qp: -1
2025-11-11T17:21:55.037575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519315361304786:2070] [node 4] ICS10 traffic start
2025-11-11T17:21:55.037624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:55.049932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519315361304784:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:55.050593Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519311066337471:2113] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:55.050827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519306771370102:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.050889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:55.050940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519306771370102:2049] [node 2] ICP22 created new session: [5:7571519315361304788:2049]
2025-11-11T17:21:55.050983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519315361304788:2049] [node 2] ICS09 handshake done sender: [5:7571519311066337471:2113] self: [5:7571519311066337484:0] peer: [2:7571519310133364155:0] socket: 168 qp: -1
2025-11-11T17:21:55.051019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519315361304788:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:55.051059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:55.071053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519311066337425:2070] [node 4] ICP35 event processing took too much time PendingConnection:278(32853324ns) {HandleHandshakeStatus:329(32851319ns) {DropIncomingHandshake:441(764ns) DropOutgoingHandshake:456(10858ns) SwitchToState:178(506ns) ProcessPendingSessionEvents:495(15811646ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15750995ns max# 15313633ns}}}
2025-11-11T17:21:55.106238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519311066337487:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:55.118605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519315361304791:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:55.133679Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519311066337474:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:55.157782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519306771370103:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.177956Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519311066337487:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.178118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519306771370101:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:55.181751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:55.207405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519306771370103:2050] [node 3] ICP22 created new session: [5:7571519315361304792:2050]
2025-11-11T17:21:55.217997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519315361304792:2050] [node 3] ICS09 handshake done sender: [5:7571519311066337474:2114] self: [5:7571519311066337481:0] peer: [3:7571519311640480696:0] socket: 174 qp: -1
2025-11-11T17:21:55.221823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519315361304792:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:55.238129Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519309780284953:2143] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:55.110616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519315935448021:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34421
2025-11-11T17:21:54.950157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519310133364172:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38695
2025-11-11T17:21:54.985838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519310473483552:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
2025-11-11T17:21:55.238237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:55.231444Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519313000088855:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.116509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519315935448018:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:55.245739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:55.231611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519304410154168:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:54.987514Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519310473483552:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.950293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519310133364173:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38695
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:55.248311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519309780284937:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(137657305ns) {HandleHandshakeStatus:329(137652726ns) {DropIncomingHandshake:441(612ns) DropOutgoingHandshake:456(812ns) SwitchToState:178(464ns) ProcessPendingSessionEvents:495(559160ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 503422ns max# 61306ns}}}
2025-11-11T17:21:55.116608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:55.260782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519306771370103:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(102228910ns) {HandleHandshakeStatus:329(102225364ns) {DropIncomingHandshake:441(730ns) DropOutgoingHandshake:456(5179ns) SwitchToState:178(1133ns) ProcessPendingSessionEvents:495(13594550ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13521828ns max# 13118995ns}}}
2025-11-11T17:21:54.951041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364172:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.960816Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364169:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.961721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:54.963314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519305838396787:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(72181040ns) {HandleHandshakeStatus:329(72179081ns) {DropIncomingHandshake:441(655ns) DropOutgoingHandshake:456(10550ns) SwitchToState:178(503ns) ProcessPendingSessionEvents:495(515038ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 457375ns max# 66736ns}}}
2025-11-11T17:21:54.969628Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364160:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.969669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519310133364160:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:54.998311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364173:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.998478Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519310133364160:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:54.998807Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519310133364156:2112] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:55.038300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519310133364147:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(27834421ns) {IncomingHandshake:261(27831780ns) {DropIncomingHandshake:441(1628ns) PrepareNewSessionHandshake:476(10099ns) {SwitchToState:178(4602ns)} GenerateSessionVirtualId:988(944ns) IssueIncomingHandshakeReply:193(7670ns)}}
2025-11-11T17:21:55.053776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519310133364146:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.053896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:55.053951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519310133364146:2107] [node 5] ICP22 created new session: [2:7571519314428331475:2107]
2025-11-11T17:21:55.054005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519314428331475:2107] [node 5] ICS09 handshake done sender: [2:7571519310133364156:2112] self: [2:7571519310133364155:0] peer: [5:7571519311066337484:0] socket: 164 qp: -1
2025-11-11T17:21:55.054040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519314428331475:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:55.054104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:55.054600Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519310133364167:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.054636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519310133364167:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.081130Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519310133364167:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:55.094081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519310133364147:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.094186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:55.094241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519310133364147:2109] [node 4] ICP22 created new session: [2:7571519314428331479:2109]
2025-11-11T17:21:55.094299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519314428331479:2109] [node 4] ICS09 handshake done sender: [2:7571519310133364167:2115] self: [2:7571519310133364162:0] peer: [4:7571519310473483551:0] socket: 157 qp: -1
2025-11-11T17:21:55.094334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519314428331479:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:55.094367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:55.097877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519310133364148:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(23978313ns) {IncomingHandshake:261(14320421ns) {DropIncomingHandshake:441(581ns) PrepareNewSessionHandshake:476(13572893ns) {SwitchToState:178(4226ns)} GenerateSessionVirtualId:988(644ns) IssueIncomingHandshakeReply:193(6553ns)}}
2025-11-11T17:21:55.111518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519310133364148:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.140089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:55.156986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519310133364148:2108] [node 6] ICP22 created new session: [2:7571519314428331481:2108]
2025-11-11T17:21:55.169759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519314428331481:2108] [node 6] ICS09 handshake done sender: [2:7571519310133364160:2114] self: [2:7571519310133364159:0] peer: [6:7571519308705121548:0] socket: 184 qp: -1
2025-11-11T17:21:55.169816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519314428331481:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:55.169920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:55.204988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519310133364148:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(92530559ns) {HandleHandshakeStatus:329(92528033ns) {DropIncomingHandshake:441(24120ns) DropOutgoingHandshake:456(578ns) SwitchToState:178(896ns) ProcessPendingSessionEvents:495(591857ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 534210ns max# 90033ns}}}
2025-11-11T17:21:55.212544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519310133364146:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(157866775ns) {HandleHandshakeStatus:329(157863486ns) {DropIncomingHandshake:441(15123ns) DropOutgoingHandshake:456(395ns) SwitchToState:178(593ns) ProcessPendingSessionEvents:495(157393739ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 157289005ns max# 24460396ns}}}
2025-11-11T17:21:54.991529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519306178516171:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(282009683ns) {Configure:104(282000414ns) {ProcessConfigured:135(16216070ns) {StartInitialHandshake:157(16214297ns) {DropHandshakes:469(2980618ns) {DropIncomingHandshake:441(965ns) DropOutgoingHandshake:456(477ns)} GenerateSessionVirtualId:988(1324ns) PrepareNewSessionHandshake:476(2218ns) {SwitchToState:178(1143ns)}}}}}
2025-11-11T17:21:55.249886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519313000088859:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:55.130607Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519315935448019:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:55.262962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519314075252253:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:54.998480Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519310473483538:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:55.274253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519315361304795:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-11T17:21:55.136677Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519311640480698:2131] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:55.147823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519311640480693:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.157090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:55.161602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519307345513312:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.161666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:55.165848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519311640480693:2107] [node 5] ICP22 created new session: [3:7571519315935448024:2107]
2025-11-11T17:21:55.173308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519311640480689:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(106151649ns) {HandleHandshakeStatus:329(106147466ns) {DropIncomingHandshake:441(14378ns) DropOutgoingHandshake:456(304ns) SwitchToState:178(319ns) ProcessPendingSessionEvents:495(14864264ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14803197ns max# 14381827ns}}}
2025-11-11T17:21:55.181696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519307345513312:2048] [node 1] ICP22 created new session: [3:7571519315935448025:2048]
2025-11-11T17:21:55.209966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519315935448024:2107] [node 5] ICS09 handshake done sender: [3:7571519311640480701:2133] self: [3:7571519311640480696:0] peer: [5:7571519311066337481:0] socket: 173 qp: -1
2025-11-11T17:21:55.213752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519315935448024:2107] [node 5] ICS10 traffic start
2025-11-11T17:21:55.220001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519315935448025:2048] [node 1] ICS09 handshake done sender: [3:7571519311640480698:2131] self: [3:7571519311640480718:0] peer: [1:7571519309780284933:0] socket: 169 qp: -1
2025-11-11T17:21:55.222905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:55.229711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519315935448025:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:55.229778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:55.235435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519311640480693:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(101939128ns) {HandleHandshakeStatus:329(101937160ns) {DropIncomingHandshake:441(23273ns) DropOutgoingHandshake:456(412ns) SwitchToState:178(751ns) ProcessPendingSessionEvents:495(492967ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 430789ns max# 48213ns}}}
2025-11-11T17:21:55.270940Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519313000088854:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:55.294233Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519311066337487:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:55.322652Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17358) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:55.341930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519306771370101:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17358) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519311066337487:2119] held: no
2025-11-11T17:21:55.262998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519314075252253:2144] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.263679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519309780284939:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.263757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:55.263803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519309780284939:2114] [node 5] ICP22 created new session: [1:7571519314075252265:2114]
2025-11-11T17:21:55.263848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519314075252265:2114] [node 5] ICS09 handshake done sender: [1:7571519309780284953:2143] self: [1:7571519309780284952:0] peer: [5:7571519315361304793:0] socket: 198 qp: -1
2025-11-11T17:21:55.263886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519314075252265:2114] [node 5] ICS10 traffic start
2025-11-11T17:21:55.263918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:55.282130Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519314075252253:2144] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:55.325880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519309780284938:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.326008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:55.326065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519309780284938:2118] [node 6] ICP22 created new session: [1:7571519314075252268:2118]
2025-11-11T17:21:55.326112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519314075252268:2118] [node 6] ICS09 handshake done sender: [1:7571519314075252253:2144] self: [1:7571519314075252252:0] peer: [6:7571519313000088858:0] socket: 189 qp: -1
2025-11-11T17:21:55.326149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519314075252268:2118] [node 6] ICS10 traffic start
2025-11-11T17:21:55.326192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:55.289630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519308705121488:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.301644Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22065) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:55.312058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:55.312135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519308705121488:2063] [node 4] ICP22 created new session: [6:7571519313000088861:2063]
2025-11-11T17:21:55.312188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519313000088861:2063] [node 4] ICS09 handshake done sender: [6:7571519313000088854:2121] self: [6:7571519313000088857:0] peer: [4:7571519310473483545:0] socket: 217 qp: -1
2025-11-11T17:21:55.312228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519313000088861:2063] [node 4] ICS10 traffic start
2025-11-11T17:21:55.312262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:55.313989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519313000088860:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18972
2025-11-11T17:21:55.314404Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519313000088855:2122] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:55.314589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519304410154168:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.314668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:55.314708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519304410154168:2048] [node 1] ICP22 created new session: [6:7571519313000088863:2048]
2025-11-11T17:21:55.314754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519313000088863:2048] [node 1] ICS09 handshake done sender: [6:7571519313000088855:2122] self: [6:7571519313000088858:0] peer: [1:7571519314075252252:0] socket: 162 qp: -1
2025-11-11T17:21:55.314787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519313000088863:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:55.314819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:55.333701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519304410154169:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(198263921ns) {HandleHandshakeStatus:329(198261074ns) {DropIncomingHandshake:441(1806ns) DropOutgoingHandshake:456(17461ns) SwitchToState:178(657ns) ProcessPendingSessionEvents:495(23351542ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23275388ns max# 22836368ns}}}
2025-11-11T17:21:55.344866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519308705121488:2063] [node 4] ICP35 event processing took too much time PendingConnection:278(31605537ns) {HandleHandshakeStatus:329(27272897ns) {DropIncomingHandshake:441(2645ns) DropOutgoingHandshake:456(6515ns) SwitchToState:178(526ns) ProcessPendingSessionEvents:495(516793ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 454560ns max# 52616ns}}}
2025-11-11T17:21:55.354904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519308705121488:2063] [node 4] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:55.364974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519306771370101:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(41123175ns) {HandleHandshakeStatus:395(41120530ns) {LogHandshakeFail:472(14644ns) {UpdateErrorStateLog:544(5607ns)} DropOutgoingHandshake:456(4314ns) ProcessPendingSessionEvents:495(22296064ns) {unordered DropSessionEvent:512 num# 100 sum# 22268793ns max# 11959481ns}}}
2025-11-11T17:21:55.365241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519306771370101:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.365284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:55.365330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519306771370101:2048] [node 1] ICP22 created new session: [5:7571519315361304796:2048]
2025-11-11T17:21:55.365376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519315361304796:2048] [node 1] ICS09 handshake done sender: [5:7571519311066337487:2119] self: [5:7571519315361304793:0] peer: [1:7571519309780284952:0] socket: 199 qp: -1
2025-11-11T17:21:55.365412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519315361304796:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:55.366474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:55.000816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519306178516171:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:55.026887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519310473483528:2101] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.054332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:55.061297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519314768450852:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
2025-11-11T17:21:55.063680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519310473483528:2101] [node 5] ICP22 created new session: [4:7571519314768450853:2101]
2025-11-11T17:21:55.066210Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519310473483532:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:55.072074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519314768450854:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
2025-11-11T17:21:55.090608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519314768450853:2101] [node 5] ICS09 handshake done sender: [4:7571519310473483538:2114] self: [4:7571519310473483537:0] peer: [5:7571519311066337486:0] socket: 191 qp: -1
2025-11-11T17:21:55.096114Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519310473483534:2112] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:55.101944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519314768450855:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12630
2025-11-11T17:21:55.106401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519314768450853:2101] [node 5] ICS10 traffic start
2025-11-11T17:21:55.118988Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:55.366639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519305485317547:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(457408351ns) {HandleHandshakeStatus:329(457404489ns) {DropIncomingHandshake:441(16193ns) DropOutgoingHandshake:456(414ns) SwitchToState:178(1284ns) ProcessPendingSessionEvents:495(367030969ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 348535462ns max# 27221849ns}}}
2025-11-11T17:21:55.121021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:55.124009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519306178516171:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.366876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519307345513312:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(204456989ns) {HandleHandshakeStatus:329(204454518ns) {DropIncomingHandshake:441(1116ns) DropOutgoingHandshake:456(5879ns) SwitchToState:178(858ns) ProcessPendingSessionEvents:495(119578206ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 108279487ns max# 24156695ns}}}
2025-11-11T17:21:55.124094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:55.124144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519306178516171:2048] [node 1] ICP22 created new session: [4:7571519314768450858:2048]
2025-11-11T17:21:55.124197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519314768450858:2048] [node 1] ICS09 handshake done sender: [4:7571519310473483534:2112] self: [4:7571519310473483555:0] peer: [1:7571519309780284944:0] socket: 171 qp: -1
2025-11-11T17:21:55.124233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519314768450858:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:55.124267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:55.125262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519306178516173:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.125345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:55.125393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519306178516173:2050] [node 3] ICP22 created new session: [4:7571519314768450860:2050]
2025-11-11T17:21:55.125428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519314768450860:2050] [node 3] ICS09 handshake done sender: [4:7571519310473483532:2110] self: [4:7571519310473483550:0] peer: [3:7571519311640480690:0] socket: 160 qp: -1
2025-11-11T17:21:55.125460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519314768450860:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:55.125495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:55.126144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519306178516173:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:55.126609Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10135) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:55.128665Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519310473483547:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:55.136849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519306178516172:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10135) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519310473483547:2117] held: no
2025-11-11T17:21:55.181826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519306178516172:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:55.181939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:55.181977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519306178516172:2049] [node 2] ICP22 created new session: [4:7571519314768450862:2049]
2025-11-11T17:21:55.182020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519314768450862:2049] [node 2] ICS09 handshake done sender: [4:7571519310473483547:2117] self: [4:7571519310473483551:0] peer: [2:7571519310133364162:0] socket: 195 qp: -1
2025-11-11T17:21:55.182060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519314768450862:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:55.182122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:55.182438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519306178516172:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24278868ns) {HandleHandshakeStatus:329(15867351ns) {DropIncomingHandshake:441(2454ns) DropOutgoingHandshake:456(337ns) SwitchToState:178(860ns) ProcessPendingSessionEvents:495(587ns)}}
2025-11-11T17:21:55.184778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519310473483528:2101] [node 5] ICP35 event processing took too much time PendingConnection:278(157058135ns) {HandleHandshakeStatus:329(157055379ns) {DropIncomingHandshake:441(24828ns) DropOutgoingHandshake:456(545ns) SwitchToState:178(319ns) ProcessPendingSessionEvents:495(26886109ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26827876ns max# 26434558ns}}}
2025-11-11T17:21:55.224046Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519310473483546:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:55.224076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519310473483546:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:55.241669Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519310473483546:2116] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:55.261697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519310473483527:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:55.261773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:55.261816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519310473483527:2100] [node 6] ICP22 created new session: [4:7571519314768450865:2100]
2025-11-11T17:21:55.283727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519314768450865:2100] [node 6] ICS09 handshake done sender: [4:7571519310473483546:2116] self: [4:7571519310473483545:0] peer: [6:7571519313000088857:0] socket: 215 qp: -1
2025-11-11T17:21:55.287363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519314768450865:2100] [node 6] ICS10 traffic start
2025-11-11T17:21:55.309742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:55.362992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519310473483527:2100] [node 6] ICP35 event processing took too much time PendingConnection:278(107471568ns) {HandleHandshakeStatus:329(107468509ns) {DropIncomingHandshake:441(27017ns) DropOutgoingHandshake:456(430ns) SwitchToState:178(460ns) ProcessPendingSessionEvents:495(11144331ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11086456ns max# 10689634ns}}}
2025-11-11T17:21:55.368447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519305485317548:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(213828345ns) {HandleHandshakeStatus:329(213824848ns) {DropIncomingHandshake:441(20642ns) DropOutgoingHandshake:456(541ns) SwitchToState:178(419ns) ProcessPendingSessionEvents:495(105623508ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 105531647ns max# 19957315ns}}}
2025-11-11T17:21:55.468426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:55.468650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519315935448025:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-11-11T17:21:55.468692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519307345513312:2048] [node 1] ICP30 unregister session Session# [3:7571519315935448025:2048] VirtualId# [3:7571519311640480718:0]
2025-11-11T17:21:55.468726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519315935448025:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.468801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:55.468846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:55.469729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:55.469944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519310133364170:2048] [node 1] ICS01 socket: 167 reason# EndOfStream
2025-11-11T17:21:55.469992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519305838396787:2048] [node 1] ICP30 unregister session Session# [2:7571519310133364170:2048] VirtualId# [2:7571519310133364161:0]
2025-11-11T17:21:55.470022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519310133364170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.470098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:55.470139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:55.481740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:55.481769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:55.482013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519314768450858:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-11T17:21:55.482012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519315361304796:2048] [node 1] ICS01 socket: 199 reason# EndOfStream
2025-11-11T17:21:55.482054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519306178516171:2048] [node 1] ICP30 unregister session Session# [4:7571519314768450858:2048] VirtualId# [4:7571519310473483555:0]
2025-11-11T17:21:55.482088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519314768450858:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.482053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519306771370101:2048] [node 1] ICP30 unregister session Session# [5:7571519315361304796:2048] VirtualId# [5:7571519315361304793:0]
2025-11-11T17:21:55.482180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:55.482087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519315361304796:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.482233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:55.482178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:55.482229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:55.485742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:55.485989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519313000088863:2048] [node 1] ICS01 socket: 162 reason# EndOfStream
2025-11-11T17:21:55.486033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519304410154168:2048] [node 1] ICP30 unregister session Session# [6:7571519313000088863:2048] VirtualId# [6:7571519313000088858:0]
2025-11-11T17:21:55.486076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519313000088863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.486190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:55.486235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:55.657214Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:55.657443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519311640480715:2049] [node 2] ICS01 socket: 166 reason# EndOfStream
2025-11-11T17:21:55.657482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519307345513313:2049] [node 2] ICP30 unregister session Session# [3:7571519311640480715:2049] VirtualId# [3:7571519311640480709:0]
2025-11-11T17:21:55.657523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519311640480715:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.657621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:55.657659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:55.664721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:55.664956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519315361304788:2049] [node 2] ICS01 socket: 168 reason# EndOfStream
2025-11-11T17:21:55.665007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519306771370102:2049] [node 2] ICP30 unregister session Session# [5:7571519315361304788:2049] VirtualId# [5:7571519311066337484:0]
2025-11-11T17:21:55.665042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519315361304788:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.665115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:55.665153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:55.667217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:55.667510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519313000088853:2049] [node 2] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-11T17:21:55.667555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519313000088853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.667640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:55.667676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519313000088853:2049] [node 2] ICS15 start handshake
2025-11-11T17:21:55.668713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:55.668897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519314768450862:2049] [node 2] ICS01 socket: 195 reason# EndOfStream
2025-11-11T17:21:55.668936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519306178516172:2049] [node 2] ICP30 unregister session Session# [4:7571519314768450862:2049] VirtualId# [4:7571519310473483551:0]
2025-11-11T17:21:55.668971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519314768450862:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.669033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:55.669074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:55.669501Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10135) Couldn't connect to any resolved address
2025-11-11T17:21:55.669698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519304410154169:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10135) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:55.682054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519313000088853:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:55.682100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519304410154169:2049] [node 2] ICP30 unregister session Session# [6:7571519313000088853:2049] VirtualId# [6:7571519308705121548:0]
2025-11-11T17:21:55.682260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:55.682301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519304410154169:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10135) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.682317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10135) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.697114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519304410154169:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:21:55.780647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:55.780902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519315361304792:2050] [node 3] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-11T17:21:55.780934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519315361304792:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.781002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:55.781037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519315361304792:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:55.781758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:55.781942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519314768450860:2050] [node 3] ICS01 socket: 160 reason# EndOfStream
2025-11-11T17:21:55.781981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519306178516173:2050] [node 3] ICP30 unregister session Session# [4:7571519314768450860:2050] VirtualId# [4:7571519310473483550:0]
2025-11-11T17:21:55.782015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519314768450860:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.782081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:55.782120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:55.783072Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address
2025-11-11T17:21:55.783326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519306771370103:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:55.783389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519315361304792:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:55.783427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519306771370103:2050] [node 3] ICP30 unregister session Session# [5:7571519315361304792:2050] VirtualId# [5:7571519311066337481:0]
2025-11-11T17:21:55.783533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:55.783583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519306771370103:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.783598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.794902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519306771370103:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:55.797753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:55.797970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519313000088850:2050] [node 3] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-11T17:21:55.798004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519313000088850:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.798111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:55.798150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519313000088850:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:55.799880Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address
2025-11-11T17:21:55.800193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519304410154170:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:55.800236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519313000088850:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:55.800268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519304410154170:2050] [node 3] ICP30 unregister session Session# [6:7571519313000088850:2050] VirtualId# [6:7571519308705121545:0]
2025-11-11T17:21:55.800353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:55.800405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519304410154170:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.800422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.813702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519304410154170:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:55.917759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:55.917999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519315361304786:2070] [node 4] ICS01 socket: 181 reason# EndOfStream
2025-11-11T17:21:55.918043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519311066337425:2070] [node 4] ICP30 unregister session Session# [5:7571519315361304786:2070] VirtualId# [5:7571519311066337486:0]
2025-11-11T17:21:55.918076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519315361304786:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.918152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:55.918193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:55.917759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:55.918005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519313000088861:2063] [node 4] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-11T17:21:55.918042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519313000088861:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:55.918131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:55.918172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519313000088861:2063] [node 4] ICS15 start handshake
2025-11-11T17:21:55.919020Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22065) Couldn't connect to any resolved address
2025-11-11T17:21:55.919275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519308705121488:2063] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:55.919372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519313000088861:2063] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:55.919439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519308705121488:2063] [node 4] ICP30 unregister session Session# [6:7571519313000088861:2063] VirtualId# [6:7571519313000088857:0]
2025-11-11T17:21:55.919584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:55.919633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519308705121488:2063] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.919652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:55.933683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519308705121488:2063] [node 4] ICP33 wake up from error state
2025-11-11T17:21:56.050270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:56.050501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519313000088849:2067] [node 5] ICS01 socket: 151 reason# EndOfStream
2025-11-11T17:21:56.050540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519308705121490:2067] [node 5] ICP30 unregister session Session# [6:7571519313000088849:2067] VirtualId# [6:7571519308705121537:0]
2025-11-11T17:21:56.050580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519313000088849:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:56.050662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:56.050702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:56.165884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519321109032004:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:56.165940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519321109032005:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:56.187939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519320561984229:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:56.187985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519320561984230:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:56.221923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519319788843636:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:56.221971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519319788843637:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:56.236214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519317649109801:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:56.236253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519317649109802:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:56.236281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519317649109803:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:56.275427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519317318510885:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:56.275486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519317318510886:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:56.275514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519317318510887:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:56.316362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519317144946009:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:56.316420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519317144946010:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:56.316465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519317144946011:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:56.845667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519317649109863:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:56.849144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519317318510938:2091] [node 4] ICP01 ready to work
2025-11-11T17:21:57.002772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519320561984301:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:56.976520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519317144946070:2082] [node 5] ICP01 ready to work
2025-11-11T17:21:56.961739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519321109032067:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:57.058087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519324856951598:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:57.060673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519324856951601:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:56.981894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519317318510949:2098] [node 6] ICP01 ready to work
2025-11-11T17:21:57.053835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519325403999364:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:57.087033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519317318510885:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24560148ns) {Configure:104(24553600ns) {ProcessConfigured:135(77629ns) {StartInitialHandshake:157(74062ns) {DropHandshakes:469(2826ns) {DropIncomingHandshake:441(699ns) DropOutgoingHandshake:456(707ns)} GenerateSessionVirtualId:988(1575ns) PrepareNewSessionHandshake:476(1519ns) {SwitchToState:178(798ns)}}}}}
2025-11-11T17:21:57.069874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519317144946009:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17607390ns) {Configure:104(17601642ns) {ProcessConfigured:135(6571193ns) {StartInitialHandshake:157(6568787ns) {DropHandshakes:469(1506ns) {DropIncomingHandshake:441(473ns) DropOutgoingHandshake:456(292ns)} GenerateSessionVirtualId:988(797ns) PrepareNewSessionHandshake:476(3040ns) {SwitchToState:178(283ns)}}}}}
2025-11-11T17:21:57.071988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519321439913373:2091] [node 4] ICP01 ready to work
2025-11-11T17:21:57.081823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519325403999365:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:57.066149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519319788843637:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(181473691ns) {Configure:104(181468007ns) {ProcessConfigured:135(2882432ns) {StartInitialHandshake:157(2879135ns) {DropHandshakes:469(5671ns) {DropIncomingHandshake:441(639ns) DropOutgoingHandshake:456(479ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:476(4371ns) {SwitchToState:178(1491ns)}}}}}
2025-11-11T17:21:57.102945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519317144946070:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30253387ns) {Configure:104(30248841ns) {ProcessConfigured:135(67248ns) {StartInitialHandshake:157(64608ns) {DropHandshakes:469(3455ns) {DropIncomingHandshake:441(712ns) DropOutgoingHandshake:456(858ns)} GenerateSessionVirtualId:988(1192ns) PrepareNewSessionHandshake:476(2294ns) {SwitchToState:178(741ns)}}}}}
2025-11-11T17:21:57.068537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519324083810999:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:57.111989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519317649109863:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(179955090ns) {Configure:104(179945753ns) {ProcessConfigured:135(26531409ns) {StartInitialHandshake:157(26529005ns) {DropHandshakes:469(1095ns) {DropIncomingHandshake:441(388ns) DropOutgoingHandshake:456(223ns)} GenerateSessionVirtualId:988(293ns) PrepareNewSessionHandshake:476(3958240ns) {SwitchToState:178(6186ns)}}}}}
2025-11-11T17:21:57.113081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519321944077163:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:57.158618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519321439913373:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53408816ns) {Configure:104(53404118ns) {ProcessConfigured:135(83962ns) {StartInitialHandshake:157(81245ns) {DropHandshakes:469(5827ns) {DropIncomingHandshake:441(992ns) DropOutgoingHandshake:456(554ns)} GenerateSessionVirtualId:988(1369ns) PrepareNewSessionHandshake:476(2158ns) {SwitchToState:178(753ns)}}}}}
2025-11-11T17:21:57.069029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519324083811000:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:57.174324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519320561984301:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(112836020ns) {Configure:104(112828279ns) {ProcessConfigured:135(96625ns) {StartInitialHandshake:157(93049ns) {DropHandshakes:469(4025ns) {DropIncomingHandshake:441(985ns) DropOutgoingHandshake:456(658ns)} GenerateSessionVirtualId:988(1356ns) PrepareNewSessionHandshake:476(2706ns) {SwitchToState:178(867ns)}}}}}
2025-11-11T17:21:57.069501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519324083811001:2109] [node 5] ICP01 ready to work
2025-11-11T17:21:57.146152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519317318510949:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(56699187ns) {Configure:104(56693919ns) {ProcessConfigured:135(119188ns) {StartInitialHandshake:157(116286ns) {DropHandshakes:469(3311ns) {DropIncomingHandshake:441(687ns) DropOutgoingHandshake:456(778ns)} GenerateSessionVirtualId:988(1237ns) PrepareNewSessionHandshake:476(2108ns) {SwitchToState:178(811ns)}}}}}
2025-11-11T17:21:57.182087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519317318510886:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(33302619ns) {Configure:104(33297322ns) {ProcessConfigured:135(86709ns) {StartInitialHandshake:157(83810ns) {DropHandshakes:469(2827ns) {DropIncomingHandshake:441(653ns) DropOutgoingHandshake:456(655ns)} GenerateSessionVirtualId:988(1120ns) PrepareNewSessionHandshake:476(3143ns) {SwitchToState:178(1226ns)}}}}}
2025-11-11T17:21:57.195080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519324856951598:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18105793ns) {Configure:104(18101304ns) {ProcessConfigured:135(87903ns) {StartInitialHandshake:157(84885ns) {DropHandshakes:469(3296ns) {DropIncomingHandshake:441(791ns) DropOutgoingHandshake:456(614ns)} GenerateSessionVirtualId:988(1143ns) PrepareNewSessionHandshake:476(2344ns) {SwitchToState:178(749ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.210063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913383:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.206251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519324083811001:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(124730565ns) {Configure:104(124728075ns) {ProcessConfigured:135(83950ns) {StartInitialHandshake:157(78848ns) {DropHandshakes:469(4170ns) {DropIncomingHandshake:441(1262ns) DropOutgoingHandshake:456(731ns)} GenerateSessionVirtualId:988(1231ns) PrepareNewSessionHandshake:476(2279ns) {SwitchToState:178(698ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.253788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519324083811008:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3837
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.257691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519324083811009:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3837
2025-11-11T17:21:57.258842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519324856951601:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(63208063ns) {Configure:104(63202498ns) {ProcessConfigured:135(67777ns) {StartInitialHandshake:157(64747ns) {DropHandshakes:469(3720ns) {DropIncomingHandshake:441(679ns) DropOutgoingHandshake:456(838ns)} GenerateSessionVirtualId:988(796ns) PrepareNewSessionHandshake:476(2279ns) {SwitchToState:178(964ns)}}}}}
2025-11-11T17:21:57.237925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519317649109802:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(61468026ns) {Configure:104(61460748ns) {ProcessConfigured:135(23787779ns) {StartInitialHandshake:157(23784539ns) {DropHandshakes:469(3688ns) {DropIncomingHandshake:441(686ns) DropOutgoingHandshake:456(643ns)} GenerateSessionVirtualId:988(1181ns) PrepareNewSessionHandshake:476(4841ns) {SwitchToState:178(1681ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.282312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519324083811013:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3837
2025-11-11T17:21:57.278141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519324856951610:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63240
2025-11-11T17:21:57.289946Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951610:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.286127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519317649109801:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(46204718ns) {Configure:104(46200284ns) {ProcessConfigured:135(2983867ns) {StartInitialHandshake:157(2979052ns) {DropHandshakes:469(4264ns) {DropIncomingHandshake:441(1192ns) DropOutgoingHandshake:456(568ns)} GenerateSessionVirtualId:988(946ns) PrepareNewSessionHandshake:476(4925ns) {SwitchToState:178(2311ns)}}}}}
2025-11-11T17:21:57.301725Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811008:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.351806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519325403999366:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2905
2025-11-11T17:21:57.351961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519325403999367:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2905
2025-11-11T17:21:57.345870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478259:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.352088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519325403999368:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2905
2025-11-11T17:21:57.352216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519325403999369:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2905
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.370083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913384:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.382639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519324856951615:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63240
2025-11-11T17:21:57.385938Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999366:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.386042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519321613478259:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.391696Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951600:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.391737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519324856951600:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.391289Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999368:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.390726Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519321439913383:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.395618Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999367:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.390868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519317144946070:2082] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.387628Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811013:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.417912Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519321613478251:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.417968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519321613478251:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.422187Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519321613478251:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:57.432443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519317318510949:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.432543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:57.432594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519317318510949:2098] [node 6] ICP22 created new session: [5:7571519321613478262:2098]
2025-11-11T17:21:57.432639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519321613478262:2098] [node 6] ICS09 handshake done sender: [5:7571519321613478251:2112] self: [5:7571519321613478250:0] peer: [6:7571519321439913385:0] socket: 161 qp: -1
2025-11-11T17:21:57.432688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519321613478262:2098] [node 6] ICS10 traffic start
2025-11-11T17:21:57.432724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:57.409846Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519324856951600:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.388912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519324083811015:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3837
2025-11-11T17:21:57.389040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519319788843637:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.392754Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811009:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.427027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519320561984230:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.447421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913386:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.477667Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.480202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519319788843637:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519324083811013:2117] held: no
2025-11-11T17:21:57.480919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519324083811017:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3837
2025-11-11T17:21:57.427129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:57.427185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519320561984230:2049] [node 3] ICP22 created new session: [2:7571519324856951617:2049]
2025-11-11T17:21:57.427242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519324856951617:2049] [node 3] ICS09 handshake done sender: [2:7571519324856951600:2119] self: [2:7571519324856951599:0] peer: [3:7571519324083811016:0] socket: 172 qp: -1
2025-11-11T17:21:57.427282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519324856951617:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:57.427326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:57.458741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478264:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.494494Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999369:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.498767Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519321944077163:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(105742854ns) {Configure:104(105736451ns) {ProcessConfigured:135(70345ns) {StartInitialHandshake:157(67351ns) {DropHandshakes:469(2722ns) {DropIncomingHandshake:441(585ns) DropOutgoingHandshake:456(640ns)} GenerateSessionVirtualId:988(739ns) PrepareNewSessionHandshake:476(1677ns) {SwitchToState:178(565ns)}}}}}
2025-11-11T17:21:57.500078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077170:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.506480Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811015:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.507004Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519324083811013:2117] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:57.500297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077171:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.500668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077175:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.507300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519321944077171:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.507593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519317649109802:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.517465Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519321944077170:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.517759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519317649109803:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.530281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077180:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.519599Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.546342Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811003:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.546384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519324083811003:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.505899Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951615:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.525950Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951603:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.525979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519324856951603:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.526551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519324856951620:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63240
2025-11-11T17:21:57.526827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519320561984229:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.537785Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519324856951603:2120] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:57.562306Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951620:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.562743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519320561984301:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.562789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:57.562840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519320561984301:2107] [node 4] ICP22 created new session: [2:7571519324856951623:2107]
2025-11-11T17:21:57.562913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519324856951623:2107] [node 4] ICS09 handshake done sender: [2:7571519324856951603:2120] self: [2:7571519324856951602:0] peer: [4:7571519321944077178:0] socket: 174 qp: -1
2025-11-11T17:21:57.562958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519324856951623:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:57.563001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:57.564276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519320561984301:2107] [node 4] ICP18 handshake (actor [2:7571519324856951620:2126]) is too old
2025-11-11T17:21:57.582623Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951607:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.582659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519324856951607:2122] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.584092Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519324856951607:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:57.584580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519324856951598:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.581819Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519321613478264:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.584639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:57.549722Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999375:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.582293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519317318510886:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.547089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519319788843637:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.584686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519324856951598:2108] [node 5] ICP22 created new session: [2:7571519324856951627:2108]
2025-11-11T17:21:57.570305Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.547149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:57.547199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519319788843637:2049] [node 2] ICP22 created new session: [3:7571519324083811021:2049]
2025-11-11T17:21:57.547247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519324083811021:2049] [node 2] ICS09 handshake done sender: [3:7571519324083811013:2117] self: [3:7571519324083811016:0] peer: [2:7571519324856951599:0] socket: 173 qp: -1
2025-11-11T17:21:57.547283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519324083811021:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:57.547318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:57.548125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519319788843636:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.548237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519324083811019:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3837
2025-11-11T17:21:57.549718Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519324083811003:2112] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:57.604634Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811019:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.604687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519324083810999:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.604735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:57.604787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519324083810999:2107] [node 4] ICP22 created new session: [3:7571519324083811025:2107]
2025-11-11T17:21:57.604834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519324083811025:2107] [node 4] ICS09 handshake done sender: [3:7571519324083811003:2112] self: [3:7571519324083811002:0] peer: [4:7571519321944077179:0] socket: 169 qp: -1
2025-11-11T17:21:57.604871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519324083811025:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:57.604907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:57.607737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519324083810999:2107] [node 4] ICP18 handshake (actor [3:7571519324083811019:2120]) is too old
2025-11-11T17:21:57.583902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478267:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.584731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519324856951627:2108] [node 5] ICS09 handshake done sender: [2:7571519324856951607:2122] self: [2:7571519324856951606:0] peer: [5:7571519321613478266:0] socket: 170 qp: -1
2025-11-11T17:21:57.549764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519325403999375:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.584772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519324856951627:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:57.584807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:57.613715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519317144946070:2082] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519321439913383:2115] held: no
2025-11-11T17:21:57.625824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519321613478264:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:57.642921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519317144946070:2082] [node 5] ICP35 event processing took too much time PendingConnection:278(48394466ns) {HandleHandshakeStatus:395(48391946ns) {LogHandshakeFail:472(7442334ns) {UpdateErrorStateLog:544(13106ns)} DropOutgoingHandshake:456(3521ns) ProcessPendingSessionEvents:495(28383124ns) {unordered DropSessionEvent:512 num# 100 sum# 14450679ns max# 7200672ns}}}
2025-11-11T17:21:57.643098Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519321439913384:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.643330Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519321439913383:2115] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:57.614292Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519321944077171:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:57.640040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519317318510886:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.640149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:57.640214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519317318510886:2049] [node 2] ICP22 created new session: [5:7571519321613478269:2049]
2025-11-11T17:21:57.640281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519321613478269:2049] [node 2] ICS09 handshake done sender: [5:7571519321613478264:2116] self: [5:7571519321613478266:0] peer: [2:7571519324856951606:0] socket: 162 qp: -1
2025-11-11T17:21:57.640322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519321613478269:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:57.640381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.589712Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999373:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.640618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519317649109802:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519321944077171:2115] held: no
2025-11-11T17:21:57.643542Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.589751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519325403999373:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.650956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519320561984229:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7571519324856951615:2125] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.652431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519324856951629:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63240
2025-11-11T17:21:57.661924Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519324856951615:2125] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:57.662165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519320561984229:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.662218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:57.662270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519320561984229:2048] [node 1] ICP22 created new session: [2:7571519324856951630:2048]
2025-11-11T17:21:57.647712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519317649109802:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.591055Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519325403999375:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:57.661418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519324083811027:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3837
2025-11-11T17:21:57.591516Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519325403999373:2115] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:57.591740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519321109032004:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.591807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:57.591861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519321109032004:2048] [node 2] ICP22 created new session: [1:7571519325403999389:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.647774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:57.647823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519317649109802:2049] [node 2] ICP22 created new session: [4:7571519321944077182:2049]
2025-11-11T17:21:57.665121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519324856951630:2048] [node 1] ICS09 handshake done sender: [2:7571519324856951615:2125] self: [2:7571519324856951622:0] peer: [1:7571519325403999374:0] socket: 182 qp: -1
2025-11-11T17:21:57.647869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519321944077182:2049] [node 2] ICS09 handshake done sender: [4:7571519321944077171:2115] self: [4:7571519321944077178:0] peer: [2:7571519324856951602:0] socket: 185 qp: -1
2025-11-11T17:21:57.665393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519324856951630:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:57.647903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519321944077182:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:57.665441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:57.647946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:57.648825Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519321944077175:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.648981Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.655155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519317649109803:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519321944077170:2114] held: no
2025-11-11T17:21:57.655924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077181:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.662970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077184:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.591909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519325403999389:2048] [node 2] ICS09 handshake done sender: [1:7571519325403999375:2116] self: [1:7571519325403999374:0] peer: [2:7571519324856951622:0] socket: 180 qp: -1
2025-11-11T17:21:57.663097Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519321944077170:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:57.591948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519325403999389:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:57.664423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519317649109803:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.682824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519317144946070:2082] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.682928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:57.682984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519317144946070:2082] [node 5] ICP22 created new session: [6:7571519321439913390:2082]
2025-11-11T17:21:57.683026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519321439913390:2082] [node 5] ICS09 handshake done sender: [6:7571519321439913383:2115] self: [6:7571519321439913385:0] peer: [5:7571519321613478250:0] socket: 163 qp: -1
2025-11-11T17:21:57.683069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519321439913390:2082] [node 5] ICS10 traffic start
2025-11-11T17:21:57.683111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:57.664495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:57.669666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519317649109803:2050] [node 3] ICP22 created new session: [4:7571519321944077188:2050]
2025-11-11T17:21:57.669738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519321944077188:2050] [node 3] ICS09 handshake done sender: [4:7571519321944077170:2114] self: [4:7571519321944077179:0] peer: [3:7571519324083811002:0] socket: 183 qp: -1
2025-11-11T17:21:57.669777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519321944077188:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:57.669821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:57.679143Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519321944077184:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.682536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519317649109801:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.591985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.688624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478268:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.689368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478271:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.689429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478272:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.593031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519321109032005:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.593112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:57.593152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519321109032005:2049] [node 3] ICP22 created new session: [1:7571519325403999391:2049]
2025-11-11T17:21:57.593220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519325403999391:2049] [node 3] ICS09 handshake done sender: [1:7571519325403999373:2115] self: [1:7571519325403999372:0] peer: [3:7571519324083811023:0] socket: 179 qp: -1
2025-11-11T17:21:57.593254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519325403999391:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:57.593284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:57.671390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519325403999395:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2905
2025-11-11T17:21:57.682871Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999395:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.683163Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999377:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.683187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519325403999377:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.690588Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519321613478272:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.690730Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.693546Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519325403999377:2117] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:57.691246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519317318510938:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.694240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519317144946009:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.694622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519321109032067:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.694419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913387:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.694783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913388:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.694949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913389:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.694723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:57.694782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519321109032067:2107] [node 4] ICP22 created new session: [1:7571519325403999398:2107]
2025-11-11T17:21:57.695578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519325403999398:2107] [node 4] ICS09 handshake done sender: [1:7571519325403999377:2117] self: [1:7571519325403999376:0] peer: [4:7571519321944077190:0] socket: 176 qp: -1
2025-11-11T17:21:57.695705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519325403999398:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:57.695743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:57.697170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519319788843636:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519324083811015:2118] held: no
2025-11-11T17:21:57.704200Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519321613478268:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.704339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519317318510885:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.703703Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519321439913389:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.704706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519321944077173:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.704744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519321944077173:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.705548Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.702032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519324083811015:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:57.702696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519319788843636:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.702743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:57.702822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519319788843636:2048] [node 1] ICP22 created new session: [3:7571519324083811029:2048]
2025-11-11T17:21:57.702896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519324083811029:2048] [node 1] ICS09 handshake done sender: [3:7571519324083811015:2118] self: [3:7571519324083811023:0] peer: [1:7571519325403999372:0] socket: 184 qp: -1
2025-11-11T17:21:57.702937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519324083811029:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:57.702974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.707533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519324856951632:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63240
2025-11-11T17:21:57.706710Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519321439913388:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.707225Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999371:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.708448Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519321439913387:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.707266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519325403999371:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.710433Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811007:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.710473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519324083811007:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.710889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519317649109801:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519321944077184:2120] held: no
2025-11-11T17:21:57.709944Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519321613478271:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.711056Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519325403999379:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.710108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519317318510887:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.711085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519325403999379:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.711895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478276:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.714478Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951632:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.716838Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519325403999371:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:57.715810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519317144946010:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.715828Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519324083811007:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.717916Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519324856951609:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.717833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077191:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.716677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519317144946011:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.717959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519324856951609:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.720517Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519321944077173:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:57.730813Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519321944077162:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.730851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519321944077162:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.731531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519321944077163:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.731624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:57.731673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519321944077163:2109] [node 5] ICP22 created new session: [4:7571519321944077193:2109]
2025-11-11T17:21:57.731724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519321944077193:2109] [node 5] ICS09 handshake done sender: [4:7571519321944077173:2116] self: [4:7571519321944077172:0] peer: [5:7571519321613478273:0] socket: 210 qp: -1
2025-11-11T17:21:57.731762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519321944077193:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:57.731795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:57.732881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519321944077184:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:57.734050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519317649109801:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.734108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:57.734157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519317649109801:2048] [node 1] ICP22 created new session: [4:7571519321944077196:2048]
2025-11-11T17:21:57.734197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519321944077196:2048] [node 1] ICS09 handshake done sender: [4:7571519321944077184:2120] self: [4:7571519321944077190:0] peer: [1:7571519325403999376:0] socket: 175 qp: -1
2025-11-11T17:21:57.734232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519321944077196:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:57.734268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:57.735908Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519321944077162:2110] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:57.736233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519317649109863:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.736274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:57.736317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519317649109863:2108] [node 6] ICP22 created new session: [4:7571519321944077198:2108]
2025-11-11T17:21:57.736363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519321944077198:2108] [node 6] ICS09 handshake done sender: [4:7571519321944077162:2110] self: [4:7571519321944077161:0] peer: [6:7571519321439913397:0] socket: 191 qp: -1
2025-11-11T17:21:57.736397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519321944077198:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:57.717443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519321439913373:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:57.719970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913396:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.730346Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519321439913384:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:57.737889Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519325403999379:2118] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:57.736429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:57.723824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478277:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.718623Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519324856951609:2123] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:57.737692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519324856951601:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.737752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:57.737798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519324856951601:2109] [node 6] ICP22 created new session: [2:7571519324856951635:2109]
2025-11-11T17:21:57.737880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519324856951635:2109] [node 6] ICS09 handshake done sender: [2:7571519324856951609:2123] self: [2:7571519324856951608:0] peer: [6:7571519321439913394:0] socket: 194 qp: -1
2025-11-11T17:21:57.737918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519324856951635:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:57.737951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:57.724297Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519324083811005:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:57.738554Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519321613478271:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:57.739374Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22073) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.746526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519317318510938:2091] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22073) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519321613478272:2120] held: no
2025-11-11T17:21:57.747183Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.724341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519324083811005:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:57.748472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519317144946009:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.748532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:57.748598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519317144946009:2048] [node 1] ICP22 created new session: [6:7571519321439913399:2048]
2025-11-11T17:21:57.748642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519321439913399:2048] [node 1] ICS09 handshake done sender: [6:7571519321439913384:2116] self: [6:7571519321439913393:0] peer: [1:7571519325403999370:0] socket: 181 qp: -1
2025-11-11T17:21:57.748684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519321439913399:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:57.748715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:57.749941Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.752537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519325403999365:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.752664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:57.752721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519325403999365:2109] [node 6] ICP22 created new session: [1:7571519325403999402:2109]
2025-11-11T17:21:57.752783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519325403999402:2109] [node 6] ICS09 handshake done sender: [1:7571519325403999371:2114] self: [1:7571519325403999370:0] peer: [6:7571519321439913393:0] socket: 178 qp: -1
2025-11-11T17:21:57.752824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519325403999402:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:57.752862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:57.725850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519324083811001:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.757743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519325403999364:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.757862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:57.757924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519325403999364:2108] [node 5] ICP22 created new session: [1:7571519325403999404:2108]
2025-11-11T17:21:57.757971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519325403999404:2108] [node 5] ICS09 handshake done sender: [1:7571519325403999379:2118] self: [1:7571519325403999378:0] peer: [5:7571519321613478274:0] socket: 177 qp: -1
2025-11-11T17:21:57.758006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519325403999404:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:57.758040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:57.725914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:57.725963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519324083811001:2109] [node 5] ICP22 created new session: [3:7571519324083811032:2109]
2025-11-11T17:21:57.726036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519324083811032:2109] [node 5] ICS09 handshake done sender: [3:7571519324083811007:2114] self: [3:7571519324083811006:0] peer: [5:7571519321613478275:0] socket: 195 qp: -1
2025-11-11T17:21:57.726077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519324083811032:2109] [node 5] ICS10 traffic start
2025-11-11T17:21:57.726112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:57.743148Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519324083811005:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:57.743489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519324083811000:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:57.743555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:57.743601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519324083811000:2108] [node 6] ICP22 created new session: [3:7571519324083811035:2108]
2025-11-11T17:21:57.743638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519324083811035:2108] [node 6] ICS09 handshake done sender: [3:7571519324083811005:2113] self: [3:7571519324083811004:0] peer: [6:7571519321439913395:0] socket: 158 qp: -1
2025-11-11T17:21:57.743679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519324083811035:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:57.743730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:57.756997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519317318510887:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.756221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913398:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.757072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:57.759476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519317318510887:2050] [node 3] ICP22 created new session: [5:7571519321613478279:2050]
2025-11-11T17:21:57.759561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519321613478279:2050] [node 3] ICS09 handshake done sender: [5:7571519321613478271:2119] self: [5:7571519321613478275:0] peer: [3:7571519324083811006:0] socket: 211 qp: -1
2025-11-11T17:21:57.760213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519321613478279:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:57.760268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:57.761482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519317318510887:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:57.762857Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519321613478272:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.766828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519317144946010:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519321439913389:2120] held: no
2025-11-11T17:21:57.767534Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.768287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519321613478278:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37874
2025-11-11T17:21:57.769476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519317318510938:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.769536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:57.769582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519317318510938:2091] [node 4] ICP22 created new session: [5:7571519321613478281:2091]
2025-11-11T17:21:57.769641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519321613478281:2091] [node 4] ICS09 handshake done sender: [5:7571519321613478272:2120] self: [5:7571519321613478273:0] peer: [4:7571519321944077172:0] socket: 212 qp: -1
2025-11-11T17:21:57.769676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519321613478281:2091] [node 4] ICS10 traffic start
2025-11-11T17:21:57.769739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:57.770468Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:57.773486Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519321439913389:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.774583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519317144946011:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519321439913388:2119] held: no
2025-11-11T17:21:57.775584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913401:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.776187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519317318510885:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519321613478268:2118] held: no
2025-11-11T17:21:57.777132Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519321613478268:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.778182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519321439913402:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40502
2025-11-11T17:21:57.778998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519317318510885:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.779059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:57.779142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519317318510885:2048] [node 1] ICP22 created new session: [5:7571519321613478283:2048]
2025-11-11T17:21:57.779183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519321613478283:2048] [node 1] ICS09 handshake done sender: [5:7571519321613478268:2118] self: [5:7571519321613478274:0] peer: [1:7571519325403999378:0] socket: 192 qp: -1
2025-11-11T17:21:57.779219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519321613478283:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:57.779257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:57.781129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519317144946010:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.781181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:57.781223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519317144946010:2049] [node 2] ICP22 created new session: [6:7571519321439913403:2049]
2025-11-11T17:21:57.781263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519321439913403:2049] [node 2] ICS09 handshake done sender: [6:7571519321439913389:2120] self: [6:7571519321439913394:0] peer: [2:7571519324856951608:0] socket: 208 qp: -1
2025-11-11T17:21:57.781296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519321439913403:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:57.781329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:57.782925Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519321439913387:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:57.784370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519321944077200:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14678
2025-11-11T17:21:57.784787Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519321439913388:2119] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:57.785136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519321439913373:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.785186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:57.785226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519321439913373:2091] [node 4] ICP22 created new session: [6:7571519321439913406:2091]
2025-11-11T17:21:57.785270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519321439913406:2091] [node 4] ICS09 handshake done sender: [6:7571519321439913387:2118] self: [6:7571519321439913397:0] peer: [4:7571519321944077161:0] socket: 206 qp: -1
2025-11-11T17:21:57.785299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519321439913406:2091] [node 4] ICS10 traffic start
2025-11-11T17:21:57.785333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:57.786695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519317144946011:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:57.786796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:57.786842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519317144946011:2050] [node 3] ICP22 created new session: [6:7571519321439913408:2050]
2025-11-11T17:21:57.786882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519321439913408:2050] [node 3] ICS09 handshake done sender: [6:7571519321439913388:2119] self: [6:7571519321439913395:0] peer: [3:7571519324083811004:0] socket: 207 qp: -1
2025-11-11T17:21:57.786916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519321439913408:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:57.786971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:57.789754Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:57.815988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:57.816220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519324856951630:2048] [node 1] ICS01 socket: 182 reason# EndOfStream
2025-11-11T17:21:57.816271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519320561984229:2048] [node 1] ICP30 unregister session Session# [2:7571519324856951630:2048] VirtualId# [2:7571519324856951622:0]
2025-11-11T17:21:57.816304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519324856951630:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.816390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:57.816431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:57.820672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:57.820926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519321613478283:2048] [node 1] ICS01 socket: 192 reason# EndOfStream
2025-11-11T17:21:57.820962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519317318510885:2048] [node 1] ICP30 unregister session Session# [5:7571519321613478283:2048] VirtualId# [5:7571519321613478274:0]
2025-11-11T17:21:57.820998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519321613478283:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.821058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:57.821126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:57.818417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:57.818660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519324083811029:2048] [node 1] ICS01 socket: 184 reason# EndOfStream
2025-11-11T17:21:57.818706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519319788843636:2048] [node 1] ICP30 unregister session Session# [3:7571519324083811029:2048] VirtualId# [3:7571519324083811023:0]
2025-11-11T17:21:57.818741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519324083811029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.818845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:57.818904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:57.821808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:57.822023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519321944077196:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-11T17:21:57.822053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519317649109801:2048] [node 1] ICP30 unregister session Session# [4:7571519321944077196:2048] VirtualId# [4:7571519321944077190:0]
2025-11-11T17:21:57.822084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519321944077196:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.822144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:57.822177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:57.825736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:57.825957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519321439913399:2048] [node 1] ICS07 socket disconnect 181 reason# EndOfStream
2025-11-11T17:21:57.825992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519321439913399:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.826076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:57.826121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519321439913399:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:57.829791Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Couldn't connect to any resolved address
2025-11-11T17:21:57.830157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519317144946009:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:57.830217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519321439913399:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:57.830261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519317144946009:2048] [node 1] ICP30 unregister session Session# [6:7571519321439913399:2048] VirtualId# [6:7571519321439913393:0]
2025-11-11T17:21:57.830313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:57.830350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519317144946009:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:57.830365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22795) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:57.841715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519317144946009:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:57.901750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:21:57.905930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:21:57.906120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519321613478269:2049] [node 2] ICS01 socket: 162 reason# EndOfStream
2025-11-11T17:21:57.906156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519317318510886:2049] [node 2] ICP30 unregister session Session# [5:7571519321613478269:2049] VirtualId# [5:7571519321613478266:0]
2025-11-11T17:21:57.906191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519321613478269:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.906283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:57.906332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:57.901996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519324083811021:2049] [node 2] ICS01 socket: 173 reason# EndOfStream
2025-11-11T17:21:57.905731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:21:57.909772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519321439913403:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-11-11T17:21:57.909813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519317144946010:2049] [node 2] ICP30 unregister session Session# [6:7571519321439913403:2049] VirtualId# [6:7571519321439913394:0]
2025-11-11T17:21:57.909846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519321439913403:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.909921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:57.909964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:57.902033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519319788843637:2049] [node 2] ICP30 unregister session Session# [3:7571519324083811021:2049] VirtualId# [3:7571519324083811016:0]
2025-11-11T17:21:57.902073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519324083811021:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.902177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:57.902218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:57.905777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:21:57.910323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519321944077182:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-11T17:21:57.910361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519317649109802:2049] [node 2] ICP30 unregister session Session# [4:7571519321944077182:2049] VirtualId# [4:7571519321944077178:0]
2025-11-11T17:21:57.910389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519321944077182:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.910449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:21:57.910498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:21:57.959200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:21:57.959450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519321944077188:2050] [node 3] ICS01 socket: 183 reason# EndOfStream
2025-11-11T17:21:57.959491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519317649109803:2050] [node 3] ICP30 unregister session Session# [4:7571519321944077188:2050] VirtualId# [4:7571519321944077179:0]
2025-11-11T17:21:57.959525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519321944077188:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.959624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:57.959671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:57.959774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:21:57.959986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519321439913408:2050] [node 3] ICS01 socket: 207 reason# EndOfStream
2025-11-11T17:21:57.960016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519317144946011:2050] [node 3] ICP30 unregister session Session# [6:7571519321439913408:2050] VirtualId# [6:7571519321439913395:0]
2025-11-11T17:21:57.960044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519321439913408:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.960105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:57.960139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:57.964997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:21:57.965261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519321613478279:2050] [node 3] ICS07 socket disconnect 211 reason# EndOfStream
2025-11-11T17:21:57.965295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519321613478279:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:57.965414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:21:57.965464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519321613478279:2050] [node 3] ICS15 start handshake
2025-11-11T17:21:57.975564Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Couldn't connect to any resolved address
2025-11-11T17:21:57.975846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519317318510887:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:57.975900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519321613478279:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:57.975935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519317318510887:2050] [node 3] ICP30 unregister session Session# [5:7571519321613478279:2050] VirtualId# [5:7571519321613478275:0]
2025-11-11T17:21:57.976048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:21:57.976074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519317318510887:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:57.976113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64782) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:57.989743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519317318510887:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:21:58.025815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:21:58.026063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519321439913406:2091] [node 4] ICS01 socket: 206 reason# EndOfStream
2025-11-11T17:21:58.026111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519321439913373:2091] [node 4] ICP30 unregister session Session# [6:7571519321439913406:2091] VirtualId# [6:7571519321439913397:0]
2025-11-11T17:21:58.026149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519321439913406:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:58.026407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:21:58.026584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519321613478281:2091] [node 4] ICS01 socket: 212 reason# EndOfStream
2025-11-11T17:21:58.026615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519317318510938:2091] [node 4] ICP30 unregister session Session# [5:7571519321613478281:2091] VirtualId# [5:7571519321613478273:0]
2025-11-11T17:21:58.026644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519321613478281:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:58.026701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:58.026735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:58.026248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:21:58.026300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:21:58.113765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:21:58.114015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519321439913390:2082] [node 5] ICS01 socket: 163 reason# EndOfStream
2025-11-11T17:21:58.114057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519317144946070:2082] [node 5] ICP30 unregister session Session# [6:7571519321439913390:2082] VirtualId# [6:7571519321439913385:0]
2025-11-11T17:21:58.114100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519321439913390:2082] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:58.114184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:21:58.114233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:21:58.289847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519326053802927:2048] [node 2] ICP01 ready to work
2025-11-11T17:21:58.289907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519326053802928:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:58.315362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519329807245027:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:58.315422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519329807245028:2049] [node 3] ICP01 ready to work
2025-11-11T17:21:58.345931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519327158195639:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:58.345978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519327158195640:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:58.368343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519327675720852:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:58.368393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519327675720853:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:58.368424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519327675720854:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:58.393687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519325654454897:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:58.393725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519325654454898:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:58.393753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519325654454899:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:58.494212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519327000187482:2048] [node 1] ICP01 ready to work
2025-11-11T17:21:58.494264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519327000187483:2049] [node 2] ICP01 ready to work
2025-11-11T17:21:58.494302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519327000187484:2050] [node 3] ICP01 ready to work
2025-11-11T17:21:58.988241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519327158195700:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:59.111591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519334102212394:2107] [node 4] ICP01 ready to work
2025-11-11T17:21:59.112009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519334102212395:2115] [node 6] ICP01 ready to work
2025-11-11T17:21:59.086239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519330348770286:2107] [node 6] ICP01 ready to work
2025-11-11T17:21:59.134069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519334102212396:2116] [node 5] ICP01 ready to work
2025-11-11T17:21:59.139444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519331453162999:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:59.151658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519327158195700:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(163059535ns) {Configure:104(163055195ns) {ProcessConfigured:135(17564135ns) {StartInitialHandshake:157(17560989ns) {DropHandshakes:469(2332ns) {DropIncomingHandshake:441(496ns) DropOutgoingHandshake:456(365ns)} GenerateSessionVirtualId:988(753ns) PrepareNewSessionHandshake:476(4883ns) {SwitchToState:178(2571ns)}}}}}
2025-11-11T17:21:59.126043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519331970688226:2108] [node 6] ICP01 ready to work
2025-11-11T17:21:59.137904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519329949422256:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:59.129692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519330348770287:2108] [node 5] ICP01 ready to work
2025-11-11T17:21:59.177742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519330348770288:2109] [node 4] ICP01 ready to work
2025-11-11T17:21:59.165793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519331295154822:2078] [node 4] ICP01 ready to work
2025-11-11T17:21:59.189799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519334102212394:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53058193ns) {Configure:104(53053791ns) {ProcessConfigured:135(56679ns) {StartInitialHandshake:157(55263ns) {DropHandshakes:469(659ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(94ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(1420ns) {SwitchToState:178(401ns)}}}}}
2025-11-11T17:21:59.147475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519329949422257:2108] [node 4] ICP01 ready to work
2025-11-11T17:21:59.210151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519331295154842:2089] [node 5] ICP01 ready to work
2025-11-11T17:21:59.239063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519326053802928:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21017973ns) {Configure:104(21009511ns) {ProcessConfigured:135(59245ns) {StartInitialHandshake:157(56483ns) {DropHandshakes:469(1840ns) {DropIncomingHandshake:441(507ns) DropOutgoingHandshake:456(205ns)} GenerateSessionVirtualId:988(577ns) PrepareNewSessionHandshake:476(2161ns) {SwitchToState:178(756ns)}}}}}
2025-11-11T17:21:59.206489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519331970688231:2110] [node 5] ICP01 ready to work
2025-11-11T17:21:59.221866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519331453163011:2109] [node 6] ICP01 ready to work
2025-11-11T17:21:59.234917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519329949422257:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(36009530ns) {Configure:104(36004344ns) {ProcessConfigured:135(8741911ns) {StartInitialHandshake:157(8738769ns) {DropHandshakes:469(2076ns) {DropIncomingHandshake:441(480ns) DropOutgoingHandshake:456(356ns)} GenerateSessionVirtualId:988(1247ns) PrepareNewSessionHandshake:476(4306ns) {SwitchToState:178(2567ns)}}}}}
2025-11-11T17:21:59.233944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519329807245027:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(252498962ns) {Configure:104(252489952ns) {ProcessConfigured:135(16895723ns) {StartInitialHandshake:157(16892847ns) {DropHandshakes:469(1990ns) {DropIncomingHandshake:441(333ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:476(5276ns) {SwitchToState:178(2001ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.259111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519327000187484:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(48582285ns) {Configure:104(48577461ns) {ProcessConfigured:135(75589ns) {StartInitialHandshake:157(72796ns) {DropHandshakes:469(2526ns) {DropIncomingHandshake:441(320ns) DropOutgoingHandshake:456(619ns)} GenerateSessionVirtualId:988(905ns) PrepareNewSessionHandshake:476(1594ns) {SwitchToState:178(745ns)}}}}}
2025-11-11T17:21:59.290673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519327000187482:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(156775091ns) {Configure:104(156763606ns) {ProcessConfigured:135(60770ns) {StartInitialHandshake:157(58943ns) {DropHandshakes:469(2759ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(341ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:476(1409ns) {SwitchToState:178(501ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.319243Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519334102212398:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.319279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519334102212398:2118] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.310226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519327675720854:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(121735977ns) {Configure:104(121731605ns) {ProcessConfigured:135(87568ns) {StartInitialHandshake:157(84571ns) {DropHandshakes:469(2556ns) {DropIncomingHandshake:441(597ns) DropOutgoingHandshake:456(414ns)} GenerateSessionVirtualId:988(1024ns) PrepareNewSessionHandshake:476(1586ns) {SwitchToState:178(615ns)}}}}}
2025-11-11T17:21:59.328720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688232:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.329029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688237:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.285775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519331453163014:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62764
2025-11-11T17:21:59.293865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519325654454898:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26661851ns) {Configure:104(26658051ns) {ProcessConfigured:135(24956754ns) {StartInitialHandshake:157(24955194ns) {DropHandshakes:469(2786ns) {DropIncomingHandshake:441(440ns) DropOutgoingHandshake:456(532ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:476(4026ns) {SwitchToState:178(431ns)}}}}}
2025-11-11T17:21:59.343354Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519331970688237:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.320544Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519334102212398:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:59.318817Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163014:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.366246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422258:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.366585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422267:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.320823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519329807245028:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.318965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519327158195640:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.367724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519327675720853:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.320871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:59.369172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519330348770289:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61018
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.321527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519331453163016:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62764
2025-11-11T17:21:59.406227Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519331453163014:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:59.398590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519331295154842:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(107413053ns) {Configure:104(107408737ns) {ProcessConfigured:135(89442ns) {StartInitialHandshake:157(87026ns) {DropHandshakes:469(2588ns) {DropIncomingHandshake:441(554ns) DropOutgoingHandshake:456(326ns)} GenerateSessionVirtualId:988(757ns) PrepareNewSessionHandshake:476(1891ns) {SwitchToState:178(899ns)}}}}}
2025-11-11T17:21:59.400060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154851:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.400243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154852:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.400306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154853:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.406543Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519331295154851:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.385750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519329949422267:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.386368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519330348770298:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61018
2025-11-11T17:21:59.417664Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519331970688232:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.320922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519329807245028:2049] [node 3] ICP22 created new session: [2:7571519334102212411:2049]
2025-11-11T17:21:59.386516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519330348770299:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61018
2025-11-11T17:21:59.405271Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770298:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.430049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519330348770304:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61018
2025-11-11T17:21:59.430240Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770289:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.434566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519327158195640:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.434645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:59.434695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519327158195640:2049] [node 2] ICP22 created new session: [3:7571519331453163021:2049]
2025-11-11T17:21:59.434749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519331453163021:2049] [node 2] ICS09 handshake done sender: [3:7571519331453163014:2115] self: [3:7571519331453163015:0] peer: [2:7571519334102212397:0] socket: 171 qp: -1
2025-11-11T17:21:59.434786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519331453163021:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:59.434846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:59.436862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519331453163017:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62764
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.409359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519325654454898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.320971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519334102212411:2049] [node 3] ICS09 handshake done sender: [2:7571519334102212398:2118] self: [2:7571519334102212397:0] peer: [3:7571519331453163015:0] socket: 166 qp: -1
2025-11-11T17:21:59.449945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519326053802927:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(172416289ns) {Configure:104(172411633ns) {ProcessConfigured:135(19605552ns) {StartInitialHandshake:157(19602991ns) {DropHandshakes:469(2645ns) {DropIncomingHandshake:441(713ns) DropOutgoingHandshake:456(562ns)} GenerateSessionVirtualId:988(781ns) PrepareNewSessionHandshake:476(5573ns) {SwitchToState:178(2618ns)}}}}}
2025-11-11T17:21:59.457675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770299:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.467816Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770291:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.467851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519330348770291:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.469653Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519330348770291:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.449744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688242:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.439551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519331453163019:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62764
2025-11-11T17:21:59.409574Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519329949422258:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.443177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519331295154822:2078] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(161109364ns) {Configure:104(161104544ns) {ProcessConfigured:135(16377355ns) {StartInitialHandshake:157(16374660ns) {DropHandshakes:469(1651ns) {DropIncomingHandshake:441(284ns) DropOutgoingHandshake:456(422ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:476(3749ns) {SwitchToState:178(1353ns)}}}}}
2025-11-11T17:21:59.490319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519326053802928:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.490405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:59.490463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519326053802928:2049] [node 3] ICP22 created new session: [1:7571519330348770310:2049]
2025-11-11T17:21:59.490516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519330348770310:2049] [node 3] ICS09 handshake done sender: [1:7571519330348770291:2111] self: [1:7571519330348770290:0] peer: [3:7571519331453163025:0] socket: 187 qp: -1
2025-11-11T17:21:59.490555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519330348770310:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:59.490596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:59.438515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422272:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.438649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519325654454899:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.442283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422274:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.439726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519331453163020:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62764
2025-11-11T17:21:59.439892Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163003:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.439918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519331453163003:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.441291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519331453163018:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62764
2025-11-11T17:21:59.442865Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519331453163003:2111] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:59.467220Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163019:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.467357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519327158195639:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.469010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519331453162999:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.469062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:59.469117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519331453162999:2108] [node 5] ICP22 created new session: [3:7571519331453163027:2108]
2025-11-11T17:21:59.469159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519331453163027:2108] [node 5] ICS09 handshake done sender: [3:7571519331453163003:2111] self: [3:7571519331453163002:0] peer: [5:7571519329949422273:0] socket: 163 qp: -1
2025-11-11T17:21:59.469196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519331453163027:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:59.469234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:59.470185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519331453163026:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62764
2025-11-11T17:21:59.470572Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519331453163019:2119] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:59.470757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519327158195639:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.470803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:59.470842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519327158195639:2048] [node 1] ICP22 created new session: [3:7571519331453163029:2048]
2025-11-11T17:21:59.470887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519331453163029:2048] [node 1] ICS09 handshake done sender: [3:7571519331453163019:2119] self: [3:7571519331453163025:0] peer: [1:7571519330348770290:0] socket: 202 qp: -1
2025-11-11T17:21:59.470921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519331453163029:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:59.470952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:59.321011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519334102212411:2049] [node 3] ICS10 traffic start
2025-11-11T17:21:59.321042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:59.338955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519329807245028:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(17222145ns) {HandleHandshakeStatus:329(17219390ns) {DropIncomingHandshake:441(274ns) DropOutgoingHandshake:456(655ns) SwitchToState:178(287ns) ProcessPendingSessionEvents:495(16845477ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16766368ns max# 8474637ns}}}
2025-11-11T17:21:59.390012Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519334102212401:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.505752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519331970688231:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(131675563ns) {Configure:104(131666867ns) {ProcessConfigured:135(10821631ns) {StartInitialHandshake:157(10819480ns) {DropHandshakes:469(1510ns) {DropIncomingHandshake:441(245ns) DropOutgoingHandshake:456(442ns)} GenerateSessionVirtualId:988(294ns) PrepareNewSessionHandshake:476(3313ns) {SwitchToState:178(1253ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.469762Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519331295154853:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.510504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519330348770313:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61018
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.514399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519327675720854:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.390056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519334102212401:2119] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.522146Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519329949422272:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.527028Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163017:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.403551Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519334102212401:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:59.494422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519331295154822:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.518362Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770293:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.562007Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519331970688230:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.562055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519331970688230:2126] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.563251Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11509) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:59.569964Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519331970688237:2130] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:59.570212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519327675720854:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11509) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7571519331970688232:2127] held: no
2025-11-11T17:21:59.571665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688246:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.571848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688247:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.527120Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163001:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.474152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519334102212414:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:21:59.474317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519334102212416:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:21:59.474368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519334102212394:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.474450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:59.474510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519334102212394:2107] [node 4] ICP22 created new session: [2:7571519334102212417:2107]
2025-11-11T17:21:59.474560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519334102212417:2107] [node 4] ICS09 handshake done sender: [2:7571519334102212401:2119] self: [2:7571519334102212400:0] peer: [4:7571519331970688241:0] socket: 170 qp: -1
2025-11-11T17:21:59.474608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519334102212417:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:59.474652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:59.476706Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519334102212403:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.476737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519334102212403:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.502330Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519334102212403:2120] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:59.534283Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519334102212405:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.534312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519334102212405:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.535824Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519334102212414:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.535889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519334102212396:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.535935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:59.535980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519334102212396:2116] [node 5] ICP22 created new session: [2:7571519334102212421:2116]
2025-11-11T17:21:59.536032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519334102212421:2116] [node 5] ICS09 handshake done sender: [2:7571519334102212403:2120] self: [2:7571519334102212402:0] peer: [5:7571519329949422271:0] socket: 169 qp: -1
2025-11-11T17:21:59.536067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519334102212421:2116] [node 5] ICS10 traffic start
2025-11-11T17:21:59.536107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:59.554665Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519334102212416:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.569143Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519334102212405:2121] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:59.578213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519334102212396:2116] [node 5] ICP18 handshake (actor [2:7571519334102212414:2123]) is too old
2025-11-11T17:21:59.527135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519331453163001:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.522275Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519329949422258:2110] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:59.518395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519330348770293:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.513191Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519331295154852:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.528194Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163018:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.526067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519329949422256:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(185728070ns) {Configure:104(185720353ns) {ProcessConfigured:135(74263ns) {StartInitialHandshake:157(70652ns) {DropHandshakes:469(4570ns) {DropIncomingHandshake:441(808ns) DropOutgoingHandshake:456(1161ns)} GenerateSessionVirtualId:988(1187ns) PrepareNewSessionHandshake:476(2542ns) {SwitchToState:178(967ns)}}}}}
2025-11-11T17:21:59.602647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519334102212395:2115] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.586849Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519331970688230:2126] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:59.513300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519327000187482:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.528881Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519331453163001:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.529112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519327158195700:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.525859Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519330348770293:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:59.587553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519327675720853:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.587615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:59.587672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519327675720853:2049] [node 2] ICP22 created new session: [4:7571519331970688250:2049]
2025-11-11T17:21:59.587717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519331970688250:2049] [node 2] ICS09 handshake done sender: [4:7571519331970688237:2130] self: [4:7571519331970688241:0] peer: [2:7571519334102212400:0] socket: 175 qp: -1
2025-11-11T17:21:59.587751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519331970688250:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:59.587788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:59.588628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519331970688226:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.588696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:59.588743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519331970688226:2108] [node 6] ICP22 created new session: [4:7571519331970688252:2108]
2025-11-11T17:21:59.588794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519331970688252:2108] [node 6] ICS09 handshake done sender: [4:7571519331970688230:2126] self: [4:7571519331970688229:0] peer: [6:7571519331295154860:0] socket: 177 qp: -1
2025-11-11T17:21:59.588825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519331970688252:2108] [node 6] ICS10 traffic start
2025-11-11T17:21:59.588851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:59.602308Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519331970688232:2127] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:59.614471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519327675720854:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.614514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:59.614559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519327675720854:2050] [node 3] ICP22 created new session: [4:7571519331970688255:2050]
2025-11-11T17:21:59.614679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519331970688255:2050] [node 3] ICS09 handshake done sender: [4:7571519331970688232:2127] self: [4:7571519331970688245:0] peer: [3:7571519331453163000:0] socket: 168 qp: -1
2025-11-11T17:21:59.614718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519331970688255:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:59.614753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:59.616002Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519331970688246:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.616601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688254:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.616669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519327675720852:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.513409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154859:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.602722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:59.526572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519325654454897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.526320Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770313:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.529156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.526883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.529188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519327158195700:2107] [node 4] ICP22 created new session: [3:7571519331453163033:2107]
2025-11-11T17:21:59.529223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519331453163033:2107] [node 4] ICS09 handshake done sender: [3:7571519331453163001:2110] self: [3:7571519331453163000:0] peer: [4:7571519331970688245:0] socket: 162 qp: -1
2025-11-11T17:21:59.529255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519331453163033:2107] [node 4] ICS10 traffic start
2025-11-11T17:21:59.529282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:59.546849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519327158195700:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(16724501ns) {HandleHandshakeStatus:329(16722271ns) {DropIncomingHandshake:441(8296ns) DropOutgoingHandshake:456(470ns) SwitchToState:178(118ns) ProcessPendingSessionEvents:495(16443503ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16363801ns max# 11589605ns}}}
2025-11-11T17:21:59.586302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519331453162999:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(25979231ns) {HandleHandshakeStatus:329(25976894ns) {DropIncomingHandshake:441(249ns) DropOutgoingHandshake:456(995ns) SwitchToState:178(185ns) ProcessPendingSessionEvents:495(25615777ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13327909ns max# 12870553ns}}}
2025-11-11T17:21:59.627248Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163013:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.627294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519331453163013:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.630221Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519331453163020:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.631163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519331453162999:2108] [node 5] ICP18 handshake (actor [3:7571519331453163018:2118]) is too old
2025-11-11T17:21:59.643324Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519331453163013:2114] [node 6] ICH04 handshake succeeded
2025-11-11T17:21:59.644895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519331453163011:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.644954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:59.645003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519331453163011:2109] [node 6] ICP22 created new session: [3:7571519331453163037:2109]
2025-11-11T17:21:59.645070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519331453163037:2109] [node 6] ICS09 handshake done sender: [3:7571519331453163013:2114] self: [3:7571519331453163012:0] peer: [6:7571519331295154865:0] socket: 205 qp: -1
2025-11-11T17:21:59.645124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519331453163037:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:59.645164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:59.605554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519334102212424:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:21:59.513713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519327000187483:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.526378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519330348770286:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.522705Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519331295154859:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.547373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519325654454899:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.526421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:59.605948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519334102212395:2115] [node 6] ICP22 created new session: [2:7571519334102212426:2115]
2025-11-11T17:21:59.622254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519334102212426:2115] [node 6] ICS09 handshake done sender: [2:7571519334102212405:2121] self: [2:7571519334102212404:0] peer: [6:7571519331295154862:0] socket: 185 qp: -1
2025-11-11T17:21:59.627084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519334102212426:2115] [node 6] ICS10 traffic start
2025-11-11T17:21:59.627156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:59.547436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:59.547490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519325654454899:2050] [node 3] ICP22 created new session: [5:7571519329949422279:2050]
2025-11-11T17:21:59.655343Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519331970688254:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.547545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519329949422279:2050] [node 3] ICS09 handshake done sender: [5:7571519329949422258:2110] self: [5:7571519329949422273:0] peer: [3:7571519331453163002:0] socket: 165 qp: -1
2025-11-11T17:21:59.547586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519329949422279:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:59.547621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:59.549850Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519329949422267:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:59.572221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519325654454898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.572276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:59.572324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519325654454898:2049] [node 2] ICP22 created new session: [5:7571519329949422281:2049]
2025-11-11T17:21:59.572382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519329949422281:2049] [node 2] ICS09 handshake done sender: [5:7571519329949422267:2115] self: [5:7571519329949422271:0] peer: [2:7571519334102212402:0] socket: 172 qp: -1
2025-11-11T17:21:59.572415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519329949422281:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:59.572483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:59.603413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422283:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.624229Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:59.632434Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519329949422272:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:59.641872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519325654454897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7571519329949422272:2116] held: no
2025-11-11T17:21:59.642468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519325654454897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.642503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:59.642558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519325654454897:2048] [node 1] ICP22 created new session: [5:7571519329949422287:2048]
2025-11-11T17:21:59.656017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519329949422287:2048] [node 1] ICS09 handshake done sender: [5:7571519329949422272:2116] self: [5:7571519329949422278:0] peer: [1:7571519330348770294:0] socket: 201 qp: -1
2025-11-11T17:21:59.656615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519329949422287:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:59.656670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.542741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519327000187484:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.660273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519334102212395:2115] [node 6] ICP35 event processing took too much time PendingConnection:278(55522020ns) {HandleHandshakeStatus:329(55520299ns) {DropIncomingHandshake:441(11046ns) DropOutgoingHandshake:456(537ns) SwitchToState:178(99ns) ProcessPendingSessionEvents:495(11828679ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11759495ns max# 9507301ns}}}
2025-11-11T17:21:59.660924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519325654454897:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18322219ns) {HandleHandshakeStatus:329(18320893ns) {DropIncomingHandshake:441(379ns) DropOutgoingHandshake:456(167ns) SwitchToState:178(201ns) ProcessPendingSessionEvents:495(1888ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.526477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519330348770286:2107] [node 6] ICP22 created new session: [1:7571519330348770315:2107]
2025-11-11T17:21:59.542848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.659828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688260:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.550134Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770295:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.550178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519330348770295:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.554240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519330348770315:2107] [node 6] ICS09 handshake done sender: [1:7571519330348770293:2112] self: [1:7571519330348770292:0] peer: [6:7571519331295154861:0] socket: 188 qp: -1
2025-11-11T17:21:59.566313Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770304:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.568983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519330348770315:2107] [node 6] ICS10 traffic start
2025-11-11T17:21:59.575633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:59.577224Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519330348770295:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:59.610505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519330348770286:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(77706490ns) {HandleHandshakeStatus:329(77702818ns) {DropIncomingHandshake:441(476ns) DropOutgoingHandshake:456(651ns) SwitchToState:178(597ns) ProcessPendingSessionEvents:495(10311178ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10245661ns max# 9805162ns}}}
2025-11-11T17:21:59.610899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519330348770287:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.610961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:59.611021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519330348770287:2108] [node 5] ICP22 created new session: [1:7571519330348770320:2108]
2025-11-11T17:21:59.611073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519330348770320:2108] [node 5] ICS09 handshake done sender: [1:7571519330348770295:2113] self: [1:7571519330348770294:0] peer: [5:7571519329949422278:0] socket: 189 qp: -1
2025-11-11T17:21:59.611111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519330348770320:2108] [node 5] ICS10 traffic start
2025-11-11T17:21:59.611150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:59.612351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519330348770286:2107] [node 6] ICP18 handshake (actor [1:7571519330348770304:2117]) is too old
2025-11-11T17:21:59.634591Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770297:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.634617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519330348770297:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.656248Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519330348770297:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:59.659605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519330348770288:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.659967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:59.660233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519330348770288:2109] [node 4] ICP22 created new session: [1:7571519330348770323:2109]
2025-11-11T17:21:59.660772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519330348770323:2109] [node 4] ICS09 handshake done sender: [1:7571519330348770297:2114] self: [1:7571519330348770296:0] peer: [4:7571519331970688258:0] socket: 210 qp: -1
2025-11-11T17:21:59.660817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519330348770323:2109] [node 4] ICS10 traffic start
2025-11-11T17:21:59.660858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.675911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422286:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.665765Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519331970688246:2133] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:59.575737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154864:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.680675Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519334102212424:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.682213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519329807245027:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.685944Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519330348770306:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.685987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519330348770306:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.622032Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519331295154853:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:59.678834Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:59.687347Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519330348770306:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:59.632194Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519331295154852:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:21:59.687232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519327675720852:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.691900Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519329949422286:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.692047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519329949422257:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.638415Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:59.695451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519326053802927:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.696655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422291:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.697429Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519329949422286:2121] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:59.697778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519329949422257:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.697837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:59.697880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519329949422257:2108] [node 4] ICP22 created new session: [5:7571519329949422292:2108]
2025-11-11T17:21:59.697927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519329949422292:2108] [node 4] ICS09 handshake done sender: [5:7571519329949422286:2121] self: [5:7571519329949422290:0] peer: [4:7571519331970688243:0] socket: 224 qp: -1
2025-11-11T17:21:59.697964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519329949422292:2108] [node 4] ICS10 traffic start
2025-11-11T17:21:59.697994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:59.687758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:59.639469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519327000187482:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.693957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519334102212430:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:21:59.639537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:59.639585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519327000187482:2048] [node 1] ICP22 created new session: [6:7571519331295154869:2048]
2025-11-11T17:21:59.639636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519331295154869:2048] [node 1] ICS09 handshake done sender: [6:7571519331295154853:2116] self: [6:7571519331295154861:0] peer: [1:7571519330348770292:0] socket: 192 qp: -1
2025-11-11T17:21:59.639672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519331295154869:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:59.639719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:59.640530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519327000187482:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:59.641561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519327000187483:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.641616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:59.642837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154866:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.648869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519327000187483:2049] [node 2] ICP22 created new session: [6:7571519331295154871:2049]
2025-11-11T17:21:59.656142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519331295154871:2049] [node 2] ICS09 handshake done sender: [6:7571519331295154852:2115] self: [6:7571519331295154862:0] peer: [2:7571519334102212404:0] socket: 191 qp: -1
2025-11-11T17:21:59.656481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519331295154871:2049] [node 2] ICS10 traffic start
2025-11-11T17:21:59.656779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:21:59.664568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154867:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.664989Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11509) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:59.668757Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519331295154851:2114] [node 4] ICH04 handshake succeeded
2025-11-11T17:21:59.682764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154868:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.684116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519327000187483:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(41833624ns) {HandleHandshakeStatus:329(41831443ns) {DropIncomingHandshake:441(432ns) DropOutgoingHandshake:456(3826ns) SwitchToState:178(656ns) ProcessPendingSessionEvents:495(14515381ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14450358ns max# 8339883ns}}}
2025-11-11T17:21:59.684201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519327000187484:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11509) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519331295154859:2118] held: no
2025-11-11T17:21:59.684755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519331295154822:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.684790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:21:59.684822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519331295154822:2078] [node 4] ICP22 created new session: [6:7571519331295154875:2078]
2025-11-11T17:21:59.684857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519331295154875:2078] [node 4] ICS09 handshake done sender: [6:7571519331295154851:2114] self: [6:7571519331295154860:0] peer: [4:7571519331970688229:0] socket: 190 qp: -1
2025-11-11T17:21:59.684883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519331295154875:2078] [node 4] ICS10 traffic start
2025-11-11T17:21:59.684910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:21:59.701874Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519331295154867:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.702331Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519331295154859:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:21:59.702664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519327000187484:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.702703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:21:59.702749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519327000187484:2050] [node 3] ICP22 created new session: [6:7571519331295154877:2050]
2025-11-11T17:21:59.702796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519331295154877:2050] [node 3] ICS09 handshake done sender: [6:7571519331295154859:2118] self: [6:7571519331295154865:0] peer: [3:7571519331453163012:0] socket: 209 qp: -1
2025-11-11T17:21:59.702828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519331295154877:2050] [node 3] ICS10 traffic start
2025-11-11T17:21:59.702859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:21:59.704944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519331295154842:2089] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:21:59.688213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519327675720852:2048] [node 1] ICP22 created new session: [4:7571519331970688261:2048]
2025-11-11T17:21:59.705687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:21:59.688266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519331970688261:2048] [node 1] ICS09 handshake done sender: [4:7571519331970688246:2133] self: [4:7571519331970688258:0] peer: [1:7571519330348770296:0] socket: 217 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.723756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519329949422289:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-11T17:21:59.725690Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519329949422276:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.705757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519326053802927:2048] [node 2] ICP22 created new session: [1:7571519330348770326:2048]
2025-11-11T17:21:59.705822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519330348770326:2048] [node 2] ICS09 handshake done sender: [1:7571519330348770306:2118] self: [1:7571519330348770303:0] peer: [2:7571519334102212429:0] socket: 215 qp: -1
2025-11-11T17:21:59.705850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519330348770326:2048] [node 2] ICS10 traffic start
2025-11-11T17:21:59.705880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.725724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519329949422276:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.730093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519334102212428:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24872
2025-11-11T17:21:59.729956Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519329949422276:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:21:59.734661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519329949422256:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.734727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:21:59.734735Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519329949422289:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.734778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519329949422256:2109] [node 6] ICP22 created new session: [5:7571519329949422295:2109]
2025-11-11T17:21:59.734871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519329949422295:2109] [node 6] ICS09 handshake done sender: [5:7571519329949422276:2118] self: [5:7571519329949422275:0] peer: [6:7571519331295154879:0] socket: 184 qp: -1
2025-11-11T17:21:59.734900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519331295154880:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-11T17:21:59.734814Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:59.734920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519329949422295:2109] [node 6] ICS10 traffic start
2025-11-11T17:21:59.734969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:21:59.688299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519331970688261:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:59.751284Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519334102212424:2125] [node 1] ICH04 handshake succeeded
2025-11-11T17:21:59.758052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519330348770288:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(46177797ns) {HandleHandshakeStatus:329(46175252ns) {DropIncomingHandshake:441(15859ns) DropOutgoingHandshake:456(530ns) SwitchToState:178(438ns) ProcessPendingSessionEvents:495(44724903ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 44472357ns max# 6036905ns}}}
2025-11-11T17:21:59.688338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:59.742107Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519331295154867:2122] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:59.691691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519331970688263:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56864
2025-11-11T17:21:59.692958Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519331970688244:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:21:59.692975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519331970688244:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:21:59.693934Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:21:59.696914Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519331970688244:2132] [node 5] ICH04 handshake succeeded
2025-11-11T17:21:59.700713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519331970688231:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:21:59.700811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:59.700859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519331970688231:2110] [node 5] ICP22 created new session: [4:7571519331970688265:2110]
2025-11-11T17:21:59.700910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519331970688265:2110] [node 5] ICS09 handshake done sender: [4:7571519331970688244:2132] self: [4:7571519331970688243:0] peer: [5:7571519329949422290:0] socket: 223 qp: -1
2025-11-11T17:21:59.700943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519331970688265:2110] [node 5] ICS10 traffic start
2025-11-11T17:21:59.700979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:59.773764Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:21:59.761828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519331295154842:2089] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.794755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519326053802927:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(46951001ns) {HandleHandshakeStatus:329(46947909ns) {DropIncomingHandshake:441(5701767ns) DropOutgoingHandshake:456(1785ns) SwitchToState:178(594ns) ProcessPendingSessionEvents:495(12305732ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12234563ns max# 7172411ns}}}
2025-11-11T17:21:59.761889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:21:59.761977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519331295154842:2089] [node 5] ICP22 created new session: [6:7571519331295154881:2089]
2025-11-11T17:21:59.762031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519331295154881:2089] [node 5] ICS09 handshake done sender: [6:7571519331295154867:2122] self: [6:7571519331295154879:0] peer: [5:7571519329949422275:0] socket: 225 qp: -1
2025-11-11T17:21:59.791515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519329807245027:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:21:59.762076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519331295154881:2089] [node 5] ICS10 traffic start
2025-11-11T17:21:59.791603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:21:59.762116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:21:59.791662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519329807245027:2048] [node 1] ICP22 created new session: [2:7571519334102212431:2048]
2025-11-11T17:21:59.791708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519334102212431:2048] [node 1] ICS09 handshake done sender: [2:7571519334102212424:2125] self: [2:7571519334102212429:0] peer: [1:7571519330348770303:0] socket: 220 qp: -1
2025-11-11T17:21:59.791749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519334102212431:2048] [node 1] ICS10 traffic start
2025-11-11T17:21:59.791789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:21:59.792579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519329807245027:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:59.796302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519329949422256:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(43535492ns) {HandleHandshakeStatus:329(43527256ns) {DropIncomingHandshake:441(850ns) DropOutgoingHandshake:456(796ns) SwitchToState:178(732ns) ProcessPendingSessionEvents:495(43001345ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 42902232ns max# 14627369ns}}}
2025-11-11T17:21:59.796406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7571519329949422256:2109] [node 6] ICP18 handshake (actor [5:7571519329949422289:2122]) is too old
2025-11-11T17:21:59.796577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519327675720852:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(80829416ns) {HandleHandshakeStatus:329(80826972ns) {DropIncomingHandshake:441(1903ns) DropOutgoingHandshake:456(5142ns) SwitchToState:178(581ns) ProcessPendingSessionEvents:495(62064463ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 61965853ns max# 14771908ns}}}
2025-11-11T17:21:59.796688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7571519327675720852:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:21:59.869741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:21:59.869742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:21:59.870074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519334102212431:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-11T17:21:59.869996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519331453163029:2048] [node 1] ICS01 socket: 202 reason# EndOfStream
2025-11-11T17:21:59.870122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519329807245027:2048] [node 1] ICP30 unregister session Session# [2:7571519334102212431:2048] VirtualId# [2:7571519334102212429:0]
2025-11-11T17:21:59.870156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519334102212431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:59.870036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519327158195639:2048] [node 1] ICP30 unregister session Session# [3:7571519331453163029:2048] VirtualId# [3:7571519331453163025:0]
2025-11-11T17:21:59.870240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:59.870081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519331453163029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:59.870283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:59.870171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:59.870220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:59.873760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:21:59.873996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519331295154869:2048] [node 1] ICS07 socket disconnect 192 reason# EndOfStream
2025-11-11T17:21:59.874017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519331295154869:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:59.874101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:59.874132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519331295154869:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:59.878147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:21:59.878830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519329949422287:2048] [node 1] ICS01 socket: 201 reason# EndOfStream
2025-11-11T17:21:59.878874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519325654454897:2048] [node 1] ICP30 unregister session Session# [5:7571519329949422287:2048] VirtualId# [5:7571519329949422278:0]
2025-11-11T17:21:59.878908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519329949422287:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:59.878989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:59.879035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:59.879157Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address
2025-11-11T17:21:59.879822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519327000187482:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:59.879879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519331295154869:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:59.879911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519327000187482:2048] [node 1] ICP30 unregister session Session# [6:7571519331295154869:2048] VirtualId# [6:7571519331295154861:0]
2025-11-11T17:21:59.879952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:59.880010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519327000187482:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:59.880049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:59.883448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:21:59.883837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519331970688261:2048] [node 1] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-11T17:21:59.883872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519331970688261:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:21:59.883986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:21:59.884043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519331970688261:2048] [node 1] ICS15 start handshake
2025-11-11T17:21:59.890160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519327000187482:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:21:59.893873Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address
2025-11-11T17:21:59.894097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519327675720852:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:21:59.894206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519331970688261:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:21:59.894260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519327675720852:2048] [node 1] ICP30 unregister session Session# [4:7571519331970688261:2048] VirtualId# [4:7571519331970688258:0]
2025-11-11T17:21:59.894384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:21:59.894422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519327675720852:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:59.894439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:21:59.909708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519327675720852:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:22:00.001739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:22:00.002016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519331453163021:2049] [node 2] ICS01 socket: 171 reason# EndOfStream
2025-11-11T17:22:00.002057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519327158195640:2049] [node 2] ICP30 unregister session Session# [3:7571519331453163021:2049] VirtualId# [3:7571519331453163015:0]
2025-11-11T17:22:00.002099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519331453163021:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.002199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:00.002253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:00.002419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:22:00.002646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519331970688250:2049] [node 2] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-11T17:22:00.002679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519331970688250:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.002757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:00.002797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519331970688250:2049] [node 2] ICS15 start handshake
2025-11-11T17:22:00.008902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:22:00.009146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519329949422281:2049] [node 2] ICS01 socket: 172 reason# EndOfStream
2025-11-11T17:22:00.009189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519325654454898:2049] [node 2] ICP30 unregister session Session# [5:7571519329949422281:2049] VirtualId# [5:7571519329949422271:0]
2025-11-11T17:22:00.009229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519329949422281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.009344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:00.009398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:00.012664Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:22:00.012921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519331295154871:2049] [node 2] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-11T17:22:00.012956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519331295154871:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.013079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:00.013132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519331295154871:2049] [node 2] ICS15 start handshake
2025-11-11T17:22:00.023439Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address
2025-11-11T17:22:00.027782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519327000187483:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:00.014527Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address
2025-11-11T17:22:00.033737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519331295154871:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:00.033826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519327000187483:2049] [node 2] ICP30 unregister session Session# [6:7571519331295154871:2049] VirtualId# [6:7571519331295154862:0]
2025-11-11T17:22:00.033992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:00.034058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519327000187483:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:00.034077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:00.022227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519327675720853:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:00.038126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519331970688250:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:00.038201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519327675720853:2049] [node 2] ICP30 unregister session Session# [4:7571519331970688250:2049] VirtualId# [4:7571519331970688241:0]
2025-11-11T17:22:00.038278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:00.038332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519327675720853:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:00.038352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:00.038673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [4:7571519327675720853:2049] [node 2] ICP35 event processing took too much time StateWork:293(20746403ns) {HandleHandshakeStatus:395(20728872ns) {LogHandshakeFail:472(19036ns) {UpdateErrorStateLog:544(3883ns)} DropOutgoingHandshake:456(4845ns) ProcessPendingSessionEvents:495(1250ns) UnregisterSession:538(54784ns) {ProcessPendingSessionEvents:495(453ns) SwitchToInitialState:201(2696ns) {SwitchToState:178(1625ns)}} TransitToErrorState:752(74253ns) {SwitchToState:178(7164ns) ProcessPendingSessionEvents:495(155ns) DropHandshakes:469(4324ns) {DropIncomingHandshake:441(1203ns) DropOutgoingHandshake:456(597ns)}}}}
2025-11-11T17:22:00.049021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519327675720853:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:22:00.044564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519327000187483:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:22:00.064524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:22:00.064697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519329949422279:2050] [node 3] ICS01 socket: 165 reason# EndOfStream
2025-11-11T17:22:00.064730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519325654454899:2050] [node 3] ICP30 unregister session Session# [5:7571519329949422279:2050] VirtualId# [5:7571519329949422273:0]
2025-11-11T17:22:00.064757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519329949422279:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.064845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:22:00.064881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:22:00.064063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:22:00.064295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519331295154877:2050] [node 3] ICS01 socket: 209 reason# EndOfStream
2025-11-11T17:22:00.064342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519327000187484:2050] [node 3] ICP30 unregister session Session# [6:7571519331295154877:2050] VirtualId# [6:7571519331295154865:0]
2025-11-11T17:22:00.064379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519331295154877:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.064449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:22:00.064592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:22:00.064489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:22:00.064797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519331970688255:2050] [node 3] ICS01 socket: 168 reason# EndOfStream
2025-11-11T17:22:00.064832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519327675720854:2050] [node 3] ICP30 unregister session Session# [4:7571519331970688255:2050] VirtualId# [4:7571519331970688245:0]
2025-11-11T17:22:00.064858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519331970688255:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.064928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:22:00.064974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:22:00.205140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:22:00.205402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519329949422292:2108] [node 4] ICS01 socket: 224 reason# EndOfStream
2025-11-11T17:22:00.205450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519329949422257:2108] [node 4] ICP30 unregister session Session# [5:7571519329949422292:2108] VirtualId# [5:7571519329949422290:0]
2025-11-11T17:22:00.205491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519329949422292:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.205628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:22:00.205693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:22:00.205155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:22:00.205474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519331295154875:2078] [node 4] ICS07 socket disconnect 190 reason# EndOfStream
2025-11-11T17:22:00.205514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519331295154875:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.205678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:22:00.205742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519331295154875:2078] [node 4] ICS15 start handshake
2025-11-11T17:22:00.209866Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8414) Couldn't connect to any resolved address
2025-11-11T17:22:00.210342Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519331295154822:2078] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8414) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:00.210414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519331295154875:2078] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:00.210457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519331295154822:2078] [node 4] ICP30 unregister session Session# [6:7571519331295154875:2078] VirtualId# [6:7571519331295154860:0]
2025-11-11T17:22:00.210513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:22:00.210549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519331295154822:2078] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:00.210565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:00.221711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519331295154822:2078] [node 4] ICP33 wake up from error state
2025-11-11T17:22:00.277787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:22:00.278020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519331295154881:2089] [node 5] ICS01 socket: 225 reason# EndOfStream
2025-11-11T17:22:00.278065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519331295154842:2089] [node 5] ICP30 unregister session Session# [6:7571519331295154881:2089] VirtualId# [6:7571519331295154879:0]
2025-11-11T17:22:00.278100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519331295154881:2089] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:00.278225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:22:00.278278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-11T17:22:00.403045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519337781901310:2048] [node 2] ICP01 ready to work
2025-11-11T17:22:00.403102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519337781901311:2049] [node 3] ICP01 ready to work
2025-11-11T17:22:00.438005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519338205562451:2048] [node 1] ICP01 ready to work
2025-11-11T17:22:00.438056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519338205562452:2049] [node 3] ICP01 ready to work
2025-11-11T17:22:00.462965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519337928617559:2048] [node 1] ICP01 ready to work
2025-11-11T17:22:00.463013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519337928617560:2049] [node 2] ICP01 ready to work
2025-11-11T17:22:00.490357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519337445480311:2048] [node 1] ICP01 ready to work
2025-11-11T17:22:00.490414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519337445480312:2049] [node 2] ICP01 ready to work
2025-11-11T17:22:00.490441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519337445480313:2050] [node 3] ICP01 ready to work
2025-11-11T17:22:00.509478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519336343247857:2048] [node 1] ICP01 ready to work
2025-11-11T17:22:00.509538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519336343247858:2049] [node 2] ICP01 ready to work
2025-11-11T17:22:00.509602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519336343247859:2050] [node 3] ICP01 ready to work
2025-11-11T17:22:00.558014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519337989399940:2048] [node 1] ICP01 ready to work
2025-11-11T17:22:00.558078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519337989399941:2049] [node 2] ICP01 ready to work
2025-11-11T17:22:00.558110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519337989399942:2050] [node 3] ICP01 ready to work
2025-11-11T17:22:01.187592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519342223584917:2108] [node 5] ICP01 ready to work
2025-11-11T17:22:01.293903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519340638215216:2103] [node 6] ICP01 ready to work
2025-11-11T17:22:01.293879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519342284367274:2079] [node 5] ICP01 ready to work
2025-11-11T17:22:01.393884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519341740447670:2108] [node 6] ICP01 ready to work
2025-11-11T17:22:01.349264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519342076868670:2107] [node 4] ICP01 ready to work
2025-11-11T17:22:01.381654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519342223584919:2107] [node 6] ICP01 ready to work
2025-11-11T17:22:01.415443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519342500529810:2107] [node 4] ICP01 ready to work
2025-11-11T17:22:01.406122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519342076868671:2109] [node 6] ICP01 ready to work
2025-11-11T17:22:01.410109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519342076868672:2108] [node 5] ICP01 ready to work
2025-11-11T17:22:01.497984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519342223584919:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(34715283ns) {Configure:104(34709782ns) {ProcessConfigured:135(7013717ns) {StartInitialHandshake:157(7003951ns) {DropHandshakes:469(2587ns) {DropIncomingHandshake:441(811ns) DropOutgoingHandshake:456(398ns)} GenerateSessionVirtualId:988(1070ns) PrepareNewSessionHandshake:476(7210ns) {SwitchToState:178(3501ns)}}}}}
2025-11-11T17:22:01.503597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519336343247857:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(274548264ns) {Configure:104(274542743ns) {ProcessConfigured:135(102051ns) {StartInitialHandshake:157(97766ns) {DropHandshakes:469(6915ns) {DropIncomingHandshake:441(874ns) DropOutgoingHandshake:456(634ns)} GenerateSessionVirtualId:988(1239ns) PrepareNewSessionHandshake:476(2174ns) {SwitchToState:178(735ns)}}}}}
2025-11-11T17:22:01.514171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519342500529811:2108] [node 5] ICP01 ready to work
2025-11-11T17:22:01.523400Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519337989399941:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(67591392ns) {Configure:104(67580495ns) {ProcessConfigured:135(95278ns) {StartInitialHandshake:157(91653ns) {DropHandshakes:469(3709ns) {DropIncomingHandshake:441(791ns) DropOutgoingHandshake:456(780ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:476(3037ns) {SwitchToState:178(1175ns)}}}}}
2025-11-11T17:22:01.517724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7571519342223584917:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(226069063ns) {Configure:104(226063555ns) {ProcessConfigured:135(34958274ns) {StartInitialHandshake:157(34953954ns) {DropHandshakes:469(3129ns) {DropIncomingHandshake:441(518ns) DropOutgoingHandshake:456(783ns)} GenerateSessionVirtualId:988(709ns) PrepareNewSessionHandshake:476(4739ns) {SwitchToState:178(1643ns)}}}}}
2025-11-11T17:22:01.551485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519342500529810:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(36734288ns) {Configure:104(36728392ns) {ProcessConfigured:135(87512ns) {StartInitialHandshake:157(84347ns) {DropHandshakes:469(3629ns) {DropIncomingHandshake:441(786ns) DropOutgoingHandshake:456(465ns)} GenerateSessionVirtualId:988(925ns) PrepareNewSessionHandshake:476(2232ns) {SwitchToState:178(663ns)}}}}}
2025-11-11T17:22:01.566203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519342500529812:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.554387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7571519342284367304:2085] [node 4] ICP01 ready to work
2025-11-11T17:22:01.568667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7571519340638215221:2109] [node 4] ICP01 ready to work
2025-11-11T17:22:01.556448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7571519341740447671:2109] [node 5] ICP01 ready to work
2025-11-11T17:22:01.557631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7571519342223584931:2109] [node 4] ICP01 ready to work
2025-11-11T17:22:01.559261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519337989399942:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(33537727ns) {Configure:104(33532087ns) {ProcessConfigured:135(79036ns) {StartInitialHandshake:157(75803ns) {DropHandshakes:469(3693ns) {DropIncomingHandshake:441(1013ns) DropOutgoingHandshake:456(700ns)} GenerateSessionVirtualId:988(1157ns) PrepareNewSessionHandshake:476(2346ns) {SwitchToState:178(839ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.597582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519337445480311:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(203185082ns) {Configure:104(203178897ns) {ProcessConfigured:135(7643918ns) {StartInitialHandshake:157(7638514ns) {DropHandshakes:469(2361ns) {DropIncomingHandshake:441(582ns) DropOutgoingHandshake:456(489ns)} GenerateSessionVirtualId:988(3448ns) PrepareNewSessionHandshake:476(3528ns) {SwitchToState:178(1459ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.602307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519342076868670:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(184196790ns) {Configure:104(184187074ns) {ProcessConfigured:135(67879ns) {StartInitialHandshake:157(64410ns) {DropHandshakes:469(2391ns) {DropIncomingHandshake:441(497ns) DropOutgoingHandshake:456(442ns)} GenerateSessionVirtualId:988(1262ns) PrepareNewSessionHandshake:476(1559ns) {SwitchToState:178(518ns)}}}}}
2025-11-11T17:22:01.618606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519338205562452:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(136291225ns) {Configure:104(136282686ns) {ProcessConfigured:135(11620611ns) {StartInitialHandshake:157(11617082ns) {DropHandshakes:469(4066ns) {DropIncomingHandshake:441(1255ns) DropOutgoingHandshake:456(570ns)} GenerateSessionVirtualId:988(1155ns) PrepareNewSessionHandshake:476(5001ns) {SwitchToState:178(2029ns)}}}}}
2025-11-11T17:22:01.618837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519342500529817:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22899
2025-11-11T17:22:01.609694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519340638215223:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:01.636147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519342284367312:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.664422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529817:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.673893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519342500529818:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22899
2025-11-11T17:22:01.619950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7571519336343247859:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(126603094ns) {Configure:104(126595536ns) {ProcessConfigured:135(22026346ns) {StartInitialHandshake:157(22022772ns) {DropHandshakes:469(3388ns) {DropIncomingHandshake:441(1137ns) DropOutgoingHandshake:456(503ns)} GenerateSessionVirtualId:988(1297ns) PrepareNewSessionHandshake:476(6206ns) {SwitchToState:178(3124ns)}}}}}
2025-11-11T17:22:01.636514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519340638215232:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.663767Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7571519337989399940:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(257501668ns) {Configure:104(257491591ns) {ProcessConfigured:135(17601638ns) {StartInitialHandshake:157(17597231ns) {DropHandshakes:469(3171ns) {DropIncomingHandshake:441(970ns) DropOutgoingHandshake:456(487ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:476(4149ns) {SwitchToState:178(1981ns)}}}}}
2025-11-11T17:22:01.674410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519337781901310:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(263445260ns) {Configure:104(263438706ns) {ProcessConfigured:135(37425608ns) {StartInitialHandshake:157(37421655ns) {DropHandshakes:469(9061005ns) {DropIncomingHandshake:441(721ns) DropOutgoingHandshake:456(2654ns)} GenerateSessionVirtualId:988(3616ns) PrepareNewSessionHandshake:476(3173ns) {SwitchToState:178(1305ns)}}}}}
2025-11-11T17:22:01.675479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519342076868681:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44320
2025-11-11T17:22:01.675653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519342076868682:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44320
2025-11-11T17:22:01.675757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519342076868683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44320
2025-11-11T17:22:01.690608Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868682:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.694624Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868681:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.705833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519342076868672:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(202833933ns) {Configure:104(202827065ns) {ProcessConfigured:135(58470229ns) {StartInitialHandshake:157(58466557ns) {DropHandshakes:469(2589ns) {DropIncomingHandshake:441(682ns) DropOutgoingHandshake:456(293ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:476(5062ns) {SwitchToState:178(2442ns)}}}}}
2025-11-11T17:22:01.706416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7571519337781901311:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(101712802ns) {Configure:104(101708221ns) {ProcessConfigured:135(11663376ns) {StartInitialHandshake:157(11659683ns) {DropHandshakes:469(3345ns) {DropIncomingHandshake:441(794ns) DropOutgoingHandshake:456(662ns)} GenerateSessionVirtualId:988(715ns) PrepareNewSessionHandshake:476(1324ns) {SwitchToState:178(414ns)}}}}}
2025-11-11T17:22:01.698538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519342223584932:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54335
2025-11-11T17:22:01.694377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519340638215233:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:01.673958Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519342284367312:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.690559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519342500529821:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22899
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.698037Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519340638215232:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.683910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519342284367315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:01.698779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519342284367274:2079] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.723496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519342284367317:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:01.710732Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529821:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.717854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584921:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.710328Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519340638215223:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.717904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519342223584921:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.722305Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519342223584921:2110] [node 5] ICH04 handshake succeeded
2025-11-11T17:22:01.722708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519342223584917:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:22:01.725469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519342223584937:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54335
2025-11-11T17:22:01.729795Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584932:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.736719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:22:01.742600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519342223584917:2108] [node 5] ICP22 created new session: [3:7571519342223584939:2108]
2025-11-11T17:22:01.773737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519342223584939:2108] [node 5] ICS09 handshake done sender: [3:7571519342223584921:2110] self: [3:7571519342223584920:0] peer: [5:7571519340638215235:0] socket: 168 qp: -1
2025-11-11T17:22:01.773834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519342223584939:2108] [node 5] ICS10 traffic start
2025-11-11T17:22:01.785820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:22:01.722111Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529818:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.782984Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519342284367317:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.710498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519336343247859:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.790759Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868683:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.746149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519342500529811:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(155206406ns) {Configure:104(155199053ns) {ProcessConfigured:135(74515ns) {StartInitialHandshake:157(71203ns) {DropHandshakes:469(1208ns) {DropIncomingHandshake:441(286ns) DropOutgoingHandshake:456(284ns)} GenerateSessionVirtualId:988(305ns) PrepareNewSessionHandshake:476(2246ns) {SwitchToState:178(720ns)}}}}}
2025-11-11T17:22:01.710554Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519340638215220:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.710580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [5:7571519340638215220:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.742073Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519340638215220:2111] [node 6] ICH04 handshake succeeded
2025-11-11T17:22:01.806035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7571519340638215216:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:22:01.793881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519337989399942:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.796230Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868676:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.790584Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584937:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.798998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519337445480312:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(229003899ns) {Configure:104(228990918ns) {ProcessConfigured:135(67276ns) {StartInitialHandshake:157(64359ns) {DropHandshakes:469(2468ns) {DropIncomingHandshake:441(594ns) DropOutgoingHandshake:456(397ns)} GenerateSessionVirtualId:988(858ns) PrepareNewSessionHandshake:476(2142ns) {SwitchToState:178(846ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.749952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7571519342500529812:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(95539096ns) {Configure:104(95531528ns) {ProcessConfigured:135(3893086ns) {StartInitialHandshake:157(3889983ns) {DropHandshakes:469(1948ns) {DropIncomingHandshake:441(424ns) DropOutgoingHandshake:456(402ns)} GenerateSessionVirtualId:988(723ns) PrepareNewSessionHandshake:476(5076ns) {SwitchToState:178(2949ns)}}}}}
2025-11-11T17:22:01.796268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519342076868676:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.836645Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519340638215233:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.793963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519337928617560:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.801873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519337445480313:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(274553795ns) {Configure:104(274540358ns) {ProcessConfigured:135(41553537ns) {StartInitialHandshake:157(41549908ns) {DropHandshakes:469(3739540ns) {DropIncomingHandshake:441(917ns) DropOutgoingHandshake:456(2892ns)} GenerateSessionVirtualId:988(2422ns) PrepareNewSessionHandshake:476(6264ns) {SwitchToState:178(1141ns)}}}}}
2025-11-11T17:22:01.795854Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519342284367315:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.836968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519336343247857:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.845878Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868679:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.841924Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529820:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.795983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519337989399940:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.795736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519342223584917:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(72106280ns) {HandleHandshakeStatus:329(72102903ns) {DropIncomingHandshake:441(269ns) DropOutgoingHandshake:456(969ns) SwitchToState:178(556ns) ProcessPendingSessionEvents:495(4618287ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4551776ns max# 4099722ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.809900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7571519341740447670:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(107820648ns) {Configure:104(107813743ns) {ProcessConfigured:135(44172518ns) {StartInitialHandshake:157(44168223ns) {DropHandshakes:469(2311ns) {DropIncomingHandshake:441(514ns) DropOutgoingHandshake:456(512ns)} GenerateSessionVirtualId:988(1108ns) PrepareNewSessionHandshake:476(4532ns) {SwitchToState:178(2340ns)}}}}}
2025-11-11T17:22:01.845913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519342076868679:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.841755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.817999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519342284367318:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:01.813872Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584927:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.841977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519342500529820:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.851157Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519342500529820:2114] [node 3] ICH04 handshake succeeded
2025-11-11T17:22:01.851632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519338205562452:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:22:01.851745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:22:01.851799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519338205562452:2049] [node 3] ICP22 created new session: [2:7571519342500529833:2049]
2025-11-11T17:22:01.851856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519342500529833:2049] [node 3] ICS09 handshake done sender: [2:7571519342500529820:2114] self: [2:7571519342500529819:0] peer: [3:7571519342223584941:0] socket: 195 qp: -1
2025-11-11T17:22:01.851891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519342500529833:2049] [node 3] ICS10 traffic start
2025-11-11T17:22:01.851953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.835811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519341740447683:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
2025-11-11T17:22:01.887579Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519342076868676:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:22:01.918269Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519342076868679:2113] [node 5] ICH04 handshake succeeded
2025-11-11T17:22:01.918413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519342076868671:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:22:01.918495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:22:01.918548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519342076868671:2109] [node 6] ICP22 created new session: [1:7571519342076868695:2109]
2025-11-11T17:22:01.918601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519342076868695:2109] [node 6] ICS09 handshake done sender: [1:7571519342076868676:2112] self: [1:7571519342076868675:0] peer: [6:7571519342284367320:0] socket: 190 qp: -1
2025-11-11T17:22:01.918640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519342076868695:2109] [node 6] ICS10 traffic start
2025-11-11T17:22:01.918702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:22:01.919943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519342076868693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44320
2025-11-11T17:22:01.920605Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868674:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.920643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519342076868674:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.931078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519342076868672:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:22:01.861760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519340638215216:2103] [node 6] ICP22 created new session: [5:7571519340638215240:2103]
2025-11-11T17:22:01.897989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519340638215240:2103] [node 6] ICS09 handshake done sender: [5:7571519340638215220:2111] self: [5:7571519340638215219:0] peer: [6:7571519342284367316:0] socket: 179 qp: -1
2025-11-11T17:22:01.910300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519340638215237:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:01.913274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519340638215241:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:01.913344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519340638215242:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:01.921682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519340638215240:2103] [node 6] ICS10 traffic start
2025-11-11T17:22:01.921757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:22:01.922265Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519340638215223:2112] [node 3] ICH04 handshake succeeded
2025-11-11T17:22:01.922928Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519340638215233:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:01.923316Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519340638215241:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.923380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519336343247859:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:22:01.923482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:22:01.923544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519336343247859:2050] [node 3] ICP22 created new session: [5:7571519340638215244:2050]
2025-11-11T17:22:01.923593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519340638215244:2050] [node 3] ICS09 handshake done sender: [5:7571519340638215223:2112] self: [5:7571519340638215235:0] peer: [3:7571519342223584920:0] socket: 172 qp: -1
2025-11-11T17:22:01.923628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519340638215244:2050] [node 3] ICS10 traffic start
2025-11-11T17:22:01.923668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:22:01.977747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519336343247857:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:01.813913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519342223584927:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.931183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:22:01.990024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.837583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519342284367321:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:01.903372Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519341740447683:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.978571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519342500529836:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22899
2025-11-11T17:22:01.862786Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [3 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:01.931239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519342076868672:2108] [node 5] ICP22 created new session: [1:7571519342076868699:2108]
2025-11-11T17:22:01.931295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519342076868699:2108] [node 5] ICS09 handshake done sender: [1:7571519342076868679:2113] self: [1:7571519342076868677:0] peer: [5:7571519340638215239:0] socket: 191 qp: -1
2025-11-11T17:22:01.861642Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 5] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16461) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.990113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519336343247857:2048] [node 1] ICP22 created new session: [5:7571519340638215246:2048]
2025-11-11T17:22:01.903529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519337445480311:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:02.018228Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529836:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.018401Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529828:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.018427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519342500529828:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:22:02.020240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519338205562451:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:02.038115Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519342500529828:2116] [node 5] ICH04 handshake succeeded
2025-11-11T17:22:01.878017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7571519337928617560:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7571519342223584937:2116] held: no
2025-11-11T17:22:01.931349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519342076868699:2108] [node 5] ICS10 traffic start
2025-11-11T17:22:01.931388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:22:01.933069Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519342076868674:2111] [node 4] ICH04 handshake succeeded
2025-11-11T17:22:01.933336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519342076868670:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:22:01.933417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:22:01.933471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519342076868670:2107] [node 4] ICP22 created new session: [1:7571519342076868702:2107]
2025-11-11T17:22:01.933524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519342076868702:2107] [node 4] ICS09 handshake done sender: [1:7571519342076868674:2111] self: [1:7571519342076868673:0] peer: [4:7571519341740447687:0] socket: 189 qp: -1
2025-11-11T17:22:01.933556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519342076868702:2107] [node 4] ICS10 traffic start
2025-11-11T17:22:01.933624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:22:02.005441Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868693:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.070962Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868680:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.071024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519342076868680:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.946321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519341740447688:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
2025-11-11T17:22:01.990167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519340638215246:2048] [node 1] ICS09 handshake done sender: [5:7571519340638215233:2117] self: [5:7571519340638215239:0] peer: [1:7571519342076868677:0] socket: 193 qp: -1
2025-11-11T17:22:01.869969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519342284367274:2079] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16461) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519342284367312:2114] held: no
2025-11-11T17:22:01.946403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519341740447689:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.878840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519342223584943:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54335
2025-11-11T17:22:01.878915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519342223584942:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54335
2025-11-11T17:22:01.879265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519342223584944:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54335
2025-11-11T17:22:01.905159Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519342223584927:2113] [node 6] ICH04 handshake succeeded
2025-11-11T17:22:01.917699Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519342223584937:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:01.940638Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584943:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.941521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519337928617560:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:01.941644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.887171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519342284367274:2079] [node 5] ICP35 event processing took too much time PendingConnection:278(24489948ns) {HandleHandshakeStatus:395(24485812ns) {LogHandshakeFail:472(18356ns) {UpdateErrorStateLog:544(6674ns)} DropOutgoingHandshake:456(9302ns) ProcessPendingSessionEvents:495(631162ns) {unordered DropSessionEvent:512 num# 100 sum# 604029ns max# 34198ns}}}
2025-11-11T17:22:01.990209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519340638215246:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:02.079130Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529814:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.951779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519342223584919:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.083544Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519342076868689:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.966614Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519342284367312:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:22:02.013732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519341740447682:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
2025-11-11T17:22:01.990245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:02.079188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519342500529814:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:22:02.083593Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519342076868689:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.951881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:22:01.990996Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:02.030154Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519341740447688:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.080319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519342500529811:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:22:01.982218Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:01.951940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519342223584919:2107] [node 6] ICP22 created new session: [3:7571519342223584949:2107]
2025-11-11T17:22:02.089925Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519342076868680:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:02.123299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7571519342076868670:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(188925852ns) {HandleHandshakeStatus:329(188922517ns) {DropIncomingHandshake:441(868ns) DropOutgoingHandshake:456(860ns) SwitchToState:178(271ns) ProcessPendingSessionEvents:495(188523426ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 185503835ns max# 24786629ns}}}
2025-11-11T17:22:02.080438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:22:02.010426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519336343247858:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.951993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519342223584949:2107] [node 6] ICS09 handshake done sender: [3:7571519342223584927:2113] self: [3:7571519342223584926:0] peer: [6:7571519342284367319:0] socket: 196 qp: -1
2025-11-11T17:22:01.983761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519342284367274:2079] [node 5] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.030309Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519341740447683:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:02.123586Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519342076868689:2118] [node 3] ICH04 handshake succeeded
2025-11-11T17:22:02.080495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519342500529811:2108] [node 5] ICP22 created new session: [2:7571519346795497136:2108]
2025-11-11T17:22:02.010893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7571519336343247857:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-11T17:22:02.058424Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519341740447682:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:01.952034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519342223584949:2107] [node 6] ICS10 traffic start
2025-11-11T17:22:01.983860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:22:02.149900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519337781901310:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.028869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519340638215216:2103] [node 6] ICP35 event processing took too much time PendingConnection:278(221923045ns) {HandleHandshakeStatus:329(221918165ns) {DropIncomingHandshake:441(17416ns) DropOutgoingHandshake:456(476ns) SwitchToState:178(934ns) ProcessPendingSessionEvents:495(66051290ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 51419116ns max# 27806806ns}}}
2025-11-11T17:22:02.054128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519344933182546:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:02.072291Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519340638215241:2119] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:02.074049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519336343247858:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.074114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:02.074167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519336343247858:2049] [node 2] ICP22 created new session: [5:7571519344933182547:2049]
2025-11-11T17:22:02.080541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519346795497136:2108] [node 5] ICS09 handshake done sender: [2:7571519342500529828:2116] self: [2:7571519342500529827:0] peer: [5:7571519344933182544:0] socket: 211 qp: -1
2025-11-11T17:22:02.058520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519337445480311:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.090370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519344933182547:2049] [node 2] ICS09 handshake done sender: [5:7571519340638215241:2119] self: [5:7571519344933182544:0] peer: [2:7571519342500529827:0] socket: 215 qp: -1
2025-11-11T17:22:02.102018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519344933182547:2049] [node 2] ICS10 traffic start
2025-11-11T17:22:02.109809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:02.127514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7571519336343247858:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(52489792ns) {HandleHandshakeStatus:329(52483192ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(8161ns) SwitchToState:178(331ns) ProcessPendingSessionEvents:495(577533ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 521862ns max# 66812ns}}}
2025-11-11T17:22:02.205940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519344933182549:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:02.217630Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [5:7571519344933182549:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.234088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7571519340638215221:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:01.952080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:22:01.983922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519342284367274:2079] [node 5] ICP22 created new session: [6:7571519342284367324:2079]
2025-11-11T17:22:02.080580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519346795497136:2108] [node 5] ICS10 traffic start
2025-11-11T17:22:01.984003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519342284367324:2079] [node 5] ICS09 handshake done sender: [6:7571519342284367312:2114] self: [6:7571519342284367316:0] peer: [5:7571519340638215219:0] socket: 183 qp: -1
2025-11-11T17:22:02.058622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:02.150030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:01.953348Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584942:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.080644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:22:02.250425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [5:7571519344933182551:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19776
2025-11-11T17:22:02.150092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519337781901310:2048] [node 2] ICP22 created new session: [1:7571519346371836002:2048]
2025-11-11T17:22:02.267065Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [5:7571519344933182549:2122] [node 4] ICH04 handshake succeeded
2025-11-11T17:22:01.953566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7571519342223584917:2108] [node 5] ICP18 handshake (actor [3:7571519342223584943:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:22:02.058695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519337445480311:2048] [node 1] ICP22 created new session: [4:7571519346035414987:2048]
2025-11-11T17:22:02.150146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519346371836002:2048] [node 2] ICS09 handshake done sender: [1:7571519342076868680:2114] self: [1:7571519342076868678:0] peer: [2:7571519346795497134:0] socket: 219 qp: -1
2025-11-11T17:22:02.058747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519346035414987:2048] [node 1] ICS09 handshake done sender: [4:7571519341740447683:2116] self: [4:7571519341740447687:0] peer: [1:7571519342076868673:0] socket: 203 qp: -1
2025-11-11T17:22:01.984050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519342284367324:2079] [node 5] ICS10 traffic start
2025-11-11T17:22:02.058784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519346035414987:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:02.058843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:01.984114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:22:02.061070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519337445480313:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:01.984572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519342284367322:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:02.042198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519337989399942:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7571519342284367317:2117] held: no
2025-11-11T17:22:02.061413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7571519337445480312:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:02.043024Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519342284367321:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.062144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519346035414986:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
2025-11-11T17:22:02.044102Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519342284367315:2116] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:02.086519Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519346035414986:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.096324Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519341740447677:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.096351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519341740447677:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:22:02.097794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519346035414992:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
2025-11-11T17:22:02.097878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519346035414993:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
2025-11-11T17:22:02.097951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [4:7571519346035414994:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41802
2025-11-11T17:22:02.113723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519341740447671:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(18753290ns)
2025-11-11T17:22:02.121850Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519341740447677:2112] [node 6] ICH04 handshake succeeded
2025-11-11T17:22:02.138289Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519341740447682:2115] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:02.145701Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519341740447688:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:22:02.153644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519337445480312:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.153745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:02.153803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519337445480312:2049] [node 2] ICP22 created new session: [4:7571519346035414997:2049]
2025-11-11T17:22:02.153847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519346035414997:2049] [node 2] ICS09 handshake done sender: [4:7571519341740447682:2115] self: [4:7571519346035414990:0] peer: [2:7571519342500529813:0] socket: 202 qp: -1
2025-11-11T17:22:02.153881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519346035414997:2049] [node 2] ICS10 traffic start
2025-11-11T17:22:02.154158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519341740447670:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.150186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519346371836002:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:02.154250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:22:02.150252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:02.154296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519341740447670:2108] [node 6] ICP22 created new session: [4:7571519346035414999:2108]
2025-11-11T17:22:02.152053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519337781901311:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.154350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519346035414999:2108] [node 6] ICS09 handshake done sender: [4:7571519341740447677:2112] self: [4:7571519341740447675:0] peer: [6:7571519346579334626:0] socket: 204 qp: -1
2025-11-11T17:22:02.154384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519346035414999:2108] [node 6] ICS10 traffic start
2025-11-11T17:22:02.152158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:22:02.152215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519337781901311:2049] [node 3] ICP22 created new session: [1:7571519346371836004:2049]
2025-11-11T17:22:02.154434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:22:02.152261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519346371836004:2049] [node 3] ICS09 handshake done sender: [1:7571519342076868689:2118] self: [1:7571519342076868687:0] peer: [3:7571519346518552249:0] socket: 187 qp: -1
2025-11-11T17:22:02.157312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:02.152301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519346371836004:2049] [node 3] ICS10 traffic start
2025-11-11T17:22:02.168566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7571519337445480313:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.152362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:22:02.168665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:22:02.206614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7571519342076868670:2107] [node 4] ICP18 handshake (actor [1:7571519342076868693:2119]) is too old
2025-11-11T17:22:02.168720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519337445480313:2050] [node 3] ICP22 created new session: [4:7571519346035415001:2050]
2025-11-11T17:22:02.214779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [1:7571519346371836006:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44320
2025-11-11T17:22:02.168769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519346035415001:2050] [node 3] ICS09 handshake done sender: [4:7571519341740447688:2117] self: [4:7571519346035414989:0] peer: [3:7571519342223584933:0] socket: 218 qp: -1
2025-11-11T17:22:02.214889Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(127.0.0.1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:02.168803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519346035415001:2050] [node 3] ICS10 traffic start
2025-11-11T17:22:02.070818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519342284367323:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:02.168839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:22:02.203615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7571519337445480312:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(49058256ns) {HandleHandshakeStatus:329(49055169ns) {DropIncomingHandshake:441(700ns) DropOutgoingHandshake:456(5581ns) SwitchToState:178(1170ns) ProcessPendingSessionEvents:495(8672271ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8607637ns max# 8116036ns}}}
2025-11-11T17:22:02.073140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519337989399940:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.225764Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519346035414992:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.073232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:02.233872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7571519341740447670:2108] [node 6] ICP18 handshake (actor [4:7571519346035414992:2120]) is too old
2025-11-11T17:22:02.073307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519337989399940:2048] [node 1] ICP22 created new session: [6:7571519346579334624:2048]
2025-11-11T17:22:02.241709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [4:7571519341740447681:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.073367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519346579334624:2048] [node 1] ICS09 handshake done sender: [6:7571519342284367315:2116] self: [6:7571519342284367320:0] peer: [1:7571519342076868675:0] socket: 192 qp: -1
2025-11-11T17:22:02.098731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519346795497135:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22899
2025-11-11T17:22:02.073409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519346579334624:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:02.073451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:02.270998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7571519340638215221:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.074905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519342284367304:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:02.099070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519346795497140:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22899
2025-11-11T17:22:02.099485Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519342284367317:2117] [node 3] ICH04 handshake succeeded
2025-11-11T17:22:02.271099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:22:02.100225Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519342500529814:2110] [node 4] ICH04 handshake succeeded
2025-11-11T17:22:02.106142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519337989399942:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.128376Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519346795497135:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.271155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7571519340638215221:2109] [node 4] ICP22 created new session: [5:7571519344933182552:2109]
2025-11-11T17:22:02.128430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519342500529810:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.106230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-11T17:22:02.106293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519337989399942:2050] [node 3] ICP22 created new session: [6:7571519346579334629:2050]
2025-11-11T17:22:02.129848Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519342500529836:2118] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:02.271209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [5:7571519344933182552:2109] [node 4] ICS09 handshake done sender: [5:7571519344933182549:2122] self: [5:7571519344933182550:0] peer: [4:7571519341740447680:0] socket: 185 qp: -1
2025-11-11T17:22:02.106344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519346579334629:2050] [node 3] ICS09 handshake done sender: [6:7571519342284367317:2117] self: [6:7571519342284367319:0] peer: [3:7571519342223584926:0] socket: 197 qp: -1
2025-11-11T17:22:02.134209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:22:02.106385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519346579334629:2050] [node 3] ICS10 traffic start
2025-11-11T17:22:02.134270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519342500529810:2107] [node 4] ICP22 created new session: [2:7571519346795497141:2107]
2025-11-11T17:22:02.271248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [5:7571519344933182552:2109] [node 4] ICS10 traffic start
2025-11-11T17:22:02.106426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [3] connected
2025-11-11T17:22:02.271288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:22:02.134332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519346795497141:2107] [node 4] ICS09 handshake done sender: [2:7571519342500529814:2110] self: [2:7571519342500529813:0] peer: [4:7571519346035414990:0] socket: 184 qp: -1
2025-11-11T17:22:02.108104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519346579334627:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:02.134371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519346795497141:2107] [node 4] ICS10 traffic start
2025-11-11T17:22:02.108168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519346579334628:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:02.134435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:22:02.153408Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519342284367321:2119] [node 4] ICH04 handshake succeeded
2025-11-11T17:22:02.146659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7571519342500529810:2107] [node 4] ICP18 handshake (actor [2:7571519346795497135:2119]) is too old
2025-11-11T17:22:02.181769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519342284367304:2085] [node 4] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.164451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519338205562451:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.205157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:22:02.164547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:02.221719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519342284367304:2085] [node 4] ICP22 created new session: [6:7571519346579334631:2085]
2025-11-11T17:22:02.164602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519338205562451:2048] [node 1] ICP22 created new session: [2:7571519346795497143:2048]
2025-11-11T17:22:02.226943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519337989399940:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(152924339ns) {HandleHandshakeStatus:329(152921012ns) {DropIncomingHandshake:441(1111ns) DropOutgoingHandshake:456(5032ns) SwitchToState:178(877ns) ProcessPendingSessionEvents:495(152463407ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 152312066ns max# 19401126ns}}}
2025-11-11T17:22:02.164653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519346795497143:2048] [node 1] ICS09 handshake done sender: [2:7571519342500529836:2118] self: [2:7571519346795497134:0] peer: [1:7571519342076868678:0] socket: 220 qp: -1
2025-11-11T17:22:02.241196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519346579334631:2085] [node 4] ICS09 handshake done sender: [6:7571519342284367321:2119] self: [6:7571519346579334626:0] peer: [4:7571519341740447675:0] socket: 206 qp: -1
2025-11-11T17:22:02.164695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519346795497143:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:02.241285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519346579334631:2085] [node 4] ICS10 traffic start
2025-11-11T17:22:02.164731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:02.257865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:22:02.301709Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519342500529830:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.278038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [6:7571519346579334628:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.301754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [2:7571519342500529830:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-11T17:22:02.285027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7571519337989399941:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:02.321246Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519342500529830:2117] [node 6] ICH04 handshake succeeded
2025-11-11T17:22:02.319291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [6:7571519346579334634:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9538
2025-11-11T17:22:02.319752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7571519342284367304:2085] [node 4] ICP35 event processing took too much time PendingConnection:278(116840669ns) {HandleHandshakeStatus:329(116836543ns) {DropIncomingHandshake:441(1064ns) DropOutgoingHandshake:456(6880ns) SwitchToState:178(642ns) ProcessPendingSessionEvents:495(651432ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 579533ns max# 87102ns}}}
2025-11-11T17:22:02.241748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [4:7571519341740447681:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-11T17:22:01.953944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519337928617560:2049] [node 2] ICP22 created new session: [3:7571519342223584947:2049]
2025-11-11T17:22:01.971611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519342223584947:2049] [node 2] ICS09 handshake done sender: [3:7571519342223584937:2116] self: [3:7571519342223584941:0] peer: [2:7571519342500529819:0] socket: 198 qp: -1
2025-11-11T17:22:01.980286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519342223584947:2049] [node 2] ICS10 traffic start
2025-11-11T17:22:02.002442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519342223584948:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54335
2025-11-11T17:22:02.027289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:02.030567Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584948:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.049873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7571519337928617559:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:02.062147Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [3:7571519342223584934:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:02.062179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [3:7571519342223584934:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-11T17:22:02.077777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519337928617560:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(124216164ns) {HandleHandshakeStatus:329(124212801ns) {DropIncomingHandshake:441(677ns) DropOutgoingHandshake:456(994ns) SwitchToState:178(666ns) ProcessPendingSessionEvents:495(2294ns)}}
2025-11-11T17:22:02.093969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7571519337928617559:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20570378ns) {IncomingHandshake:261(20568092ns) {DropIncomingHandshake:441(563ns) PrepareNewSessionHandshake:476(1395ns) {SwitchToState:178(554ns)} GenerateSessionVirtualId:988(806ns) IssueIncomingHandshakeReply:193(12485575ns) {SwitchToState:178(2423ns)}}}
2025-11-11T17:22:02.099725Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519342223584934:2115] [node 4] ICH04 handshake succeeded
2025-11-11T17:22:02.118701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [3:7571519346518552251:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54335
2025-11-11T17:22:02.135704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7571519342223584931:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.135795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-11T17:22:02.135844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519342223584931:2109] [node 4] ICP22 created new session: [3:7571519346518552252:2109]
2025-11-11T17:22:02.135889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519346518552252:2109] [node 4] ICS09 handshake done sender: [3:7571519342223584934:2115] self: [3:7571519342223584933:0] peer: [4:7571519346035414989:0] socket: 217 qp: -1
2025-11-11T17:22:02.135921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519346518552252:2109] [node 4] ICS10 traffic start
2025-11-11T17:22:02.135954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [4] connected
2025-11-11T17:22:02.186980Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [3:7571519342223584948:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:02.188202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7571519337928617559:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.188320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:02.188380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7571519337928617559:2048] [node 1] ICP22 created new session: [3:7571519346518552254:2048]
2025-11-11T17:22:02.188432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [3:7571519346518552254:2048] [node 1] ICS09 handshake done sender: [3:7571519342223584948:2120] self: [3:7571519346518552249:0] peer: [1:7571519342076868687:0] socket: 222 qp: -1
2025-11-11T17:22:02.188472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [3:7571519346518552254:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:02.188514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:02.265874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [4:7571519341740447681:2114] [node 5] ICH04 handshake succeeded
2025-11-11T17:22:02.267498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7571519341740447671:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.267613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-11T17:22:02.267674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7571519341740447671:2109] [node 5] ICP22 created new session: [4:7571519346035415005:2109]
2025-11-11T17:22:02.267726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [4:7571519346035415005:2109] [node 5] ICS09 handshake done sender: [4:7571519341740447681:2114] self: [4:7571519341740447680:0] peer: [5:7571519344933182550:0] socket: 180 qp: -1
2025-11-11T17:22:02.267768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [4:7571519346035415005:2109] [node 5] ICS10 traffic start
2025-11-11T17:22:02.267839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [5] connected
2025-11-11T17:22:02.335487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519342500529812:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-11T17:22:02.349910Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [6:7571519346579334628:2123] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:02.361841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7571519337989399941:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:02.361944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:02.362007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7571519337989399941:2049] [node 2] ICP22 created new session: [6:7571519346579334635:2049]
2025-11-11T17:22:02.362055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [6:7571519346579334635:2049] [node 2] ICS09 handshake done sender: [6:7571519346579334628:2123] self: [6:7571519346579334633:0] peer: [2:7571519342500529826:0] socket: 235 qp: -1
2025-11-11T17:22:02.362110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [6:7571519346579334635:2049] [node 2] ICS10 traffic start
2025-11-11T17:22:02.362155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:02.363676Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:02.346465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-11T17:22:02.346538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519342500529812:2109] [node 6] ICP22 created new session: [2:7571519346795497146:2109]
2025-11-11T17:22:02.346597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519346795497146:2109] [node 6] ICS09 handshake done sender: [2:7571519342500529830:2117] self: [2:7571519342500529826:0] peer: [6:7571519346579334633:0] socket: 234 qp: -1
2025-11-11T17:22:02.346647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519346795497146:2109] [node 6] ICS10 traffic start
2025-11-11T17:22:02.346706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [6] connected
2025-11-11T17:22:02.375869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7571519342500529812:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(39495605ns) {HandleHandshakeStatus:329(39490809ns) {DropIncomingHandshake:441(29769ns) DropOutgoingHandshake:456(446ns) SwitchToState:178(634ns) ProcessPendingSessionEvents:495(28034168ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27945465ns max# 10800752ns}}}
2025-11-11T17:22:02.374347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7571519337989399941:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-11T17:22:02.446304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:02.446611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519346795497143:2048] [node 1] ICS07 socket disconnect 220 reason# EndOfStream
2025-11-11T17:22:02.446644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519346795497143:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.446756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:02.446798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519346795497143:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:02.449838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 1] connection closed by peer
2025-11-11T17:22:02.449663Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
2025-11-11T17:22:02.450024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519338205562451:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:22:02.450774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519346795497143:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:02.451392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519346518552254:2048] [node 1] ICS01 socket: 222 reason# EndOfStream
2025-11-11T17:22:02.451459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519337928617559:2048] [node 1] ICP30 unregister session Session# [3:7571519346518552254:2048] VirtualId# [3:7571519346518552249:0]
2025-11-11T17:22:02.451499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519346518552254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.451608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:02.451769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:02.453570Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address
2025-11-11T17:22:02.453922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519338205562451:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.453989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519346795497143:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.454028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519338205562451:2048] [node 1] ICP30 unregister session Session# [2:7571519346795497143:2048] VirtualId# [2:7571519346795497134:0]
2025-11-11T17:22:02.454070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:02.454104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571519338205562451:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.454150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.454784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 1] connection closed by peer
2025-11-11T17:22:02.455001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519346035414987:2048] [node 1] ICS01 socket: 203 reason# EndOfStream
2025-11-11T17:22:02.455038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519337445480311:2048] [node 1] ICP30 unregister session Session# [4:7571519346035414987:2048] VirtualId# [4:7571519341740447687:0]
2025-11-11T17:22:02.455070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519346035414987:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.455149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:02.455187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:02.464315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571519338205562451:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:22:02.469792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 1] connection closed by peer
2025-11-11T17:22:02.463549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 1] connection closed by peer
2025-11-11T17:22:02.463763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519340638215246:2048] [node 1] ICS07 socket disconnect 193 reason# EndOfStream
2025-11-11T17:22:02.463789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519340638215246:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.463881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:02.463916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519340638215246:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:02.477084Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address
2025-11-11T17:22:02.477675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519336343247857:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.477735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519340638215246:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.477773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519336343247857:2048] [node 1] ICP30 unregister session Session# [5:7571519340638215246:2048] VirtualId# [5:7571519340638215239:0]
2025-11-11T17:22:02.477823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:02.477862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519336343247857:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.477878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.474727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519346579334624:2048] [node 1] ICS07 socket disconnect 192 reason# EndOfStream
2025-11-11T17:22:02.474788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519346579334624:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.474909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:02.474965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519346579334624:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:02.478482Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address
2025-11-11T17:22:02.478824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519337989399940:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.478874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519346579334624:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.489822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519336343247857:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:22:02.478916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519337989399940:2048] [node 1] ICP30 unregister session Session# [6:7571519346579334624:2048] VirtualId# [6:7571519342284367320:0]
2025-11-11T17:22:02.484492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:02.484551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519337989399940:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.484569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8365) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.493986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519337989399940:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:22:02.525863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [3 <-> 2] connection closed by peer
2025-11-11T17:22:02.526154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7571519342223584947:2049] [node 2] ICS01 socket: 198 reason# EndOfStream
2025-11-11T17:22:02.526202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7571519337928617560:2049] [node 2] ICP30 unregister session Session# [3:7571519342223584947:2049] VirtualId# [3:7571519342223584941:0]
2025-11-11T17:22:02.526236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7571519342223584947:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.526326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:02.526366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:02.533237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 2] connection closed by peer
2025-11-11T17:22:02.533490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [4:7571519346035414997:2049] [node 2] ICS07 socket disconnect 202 reason# EndOfStream
2025-11-11T17:22:02.533520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519346035414997:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.533624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:02.533677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [4:7571519346035414997:2049] [node 2] ICS15 start handshake
2025-11-11T17:22:02.534248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 2] connection closed by peer
2025-11-11T17:22:02.534434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519344933182547:2049] [node 2] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-11T17:22:02.534461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519344933182547:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.534528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:02.534565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519344933182547:2049] [node 2] ICS15 start handshake
2025-11-11T17:22:02.535727Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [4 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address
2025-11-11T17:22:02.536074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7571519337445480312:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.536127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519346035414997:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.536160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519337445480312:2049] [node 2] ICP30 unregister session Session# [4:7571519346035414997:2049] VirtualId# [4:7571519346035414990:0]
2025-11-11T17:22:02.536256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:02.536278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7571519337445480312:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.536289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.536595Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 2] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address
2025-11-11T17:22:02.536797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519336343247858:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.536831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519344933182547:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.536849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519336343247858:2049] [node 2] ICP30 unregister session Session# [5:7571519344933182547:2049] VirtualId# [5:7571519344933182544:0]
2025-11-11T17:22:02.536947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:02.536973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519336343247858:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.536982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29529) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.537817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 2] connection closed by peer
2025-11-11T17:22:02.538035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519346579334635:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-11T17:22:02.538076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519337989399941:2049] [node 2] ICP30 unregister session Session# [6:7571519346579334635:2049] VirtualId# [6:7571519346579334633:0]
2025-11-11T17:22:02.538116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519346579334635:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.538240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:02.538285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:02.549708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7571519337445480312:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:22:02.549827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519336343247858:2049] [node 2] ICP33 wake up from error state
2025-11-11T17:22:02.637808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 3] connection closed by peer
2025-11-11T17:22:02.638045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519346579334629:2050] [node 3] ICS01 socket: 197 reason# EndOfStream
2025-11-11T17:22:02.638085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519337989399942:2050] [node 3] ICP30 unregister session Session# [6:7571519346579334629:2050] VirtualId# [6:7571519342284367319:0]
2025-11-11T17:22:02.637766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 3] connection closed by peer
2025-11-11T17:22:02.638118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519346579334629:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.638199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:22:02.638244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:22:02.638047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519340638215244:2050] [node 3] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-11T17:22:02.638355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [4 <-> 3] connection closed by peer
2025-11-11T17:22:02.638092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519340638215244:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.638546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7571519346035415001:2050] [node 3] ICS01 socket: 218 reason# EndOfStream
2025-11-11T17:22:02.638223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:22:02.638584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7571519337445480313:2050] [node 3] ICP30 unregister session Session# [4:7571519346035415001:2050] VirtualId# [4:7571519346035414989:0]
2025-11-11T17:22:02.638269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519340638215244:2050] [node 3] ICS15 start handshake
2025-11-11T17:22:02.638610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7571519346035415001:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.638669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [3] disconnected
2025-11-11T17:22:02.638704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:22:02.639264Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 3] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16340) Couldn't connect to any resolved address
2025-11-11T17:22:02.639605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519336343247859:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.639867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519340638215244:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.639922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519336343247859:2050] [node 3] ICP30 unregister session Session# [5:7571519340638215244:2050] VirtualId# [5:7571519340638215235:0]
2025-11-11T17:22:02.639967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [3] session destroyed
2025-11-11T17:22:02.640002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519336343247859:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.640018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.649476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519336343247859:2050] [node 3] ICP33 wake up from error state
2025-11-11T17:22:02.721313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [5 <-> 4] connection closed by peer
2025-11-11T17:22:02.721573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [5:7571519344933182552:2109] [node 4] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-11T17:22:02.721622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7571519344933182552:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.721749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:22:02.721797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [5:7571519344933182552:2109] [node 4] ICS15 start handshake
2025-11-11T17:22:02.722571Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [5 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address
2025-11-11T17:22:02.722893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7571519340638215221:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.722947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7571519344933182552:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.722996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7571519340638215221:2109] [node 4] ICP30 unregister session Session# [5:7571519344933182552:2109] VirtualId# [5:7571519344933182550:0]
2025-11-11T17:22:02.723142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:22:02.723212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7571519340638215221:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.723258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.726164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 4] connection closed by peer
2025-11-11T17:22:02.731648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [6:7571519346579334631:2085] [node 4] ICS07 socket disconnect 206 reason# EndOfStream
2025-11-11T17:22:02.731700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519346579334631:2085] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.731840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [4] disconnected
2025-11-11T17:22:02.731885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [6:7571519346579334631:2085] [node 4] ICS15 start handshake
2025-11-11T17:22:02.733010Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [6 <-> 4] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address
2025-11-11T17:22:02.733293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7571519342284367304:2085] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:02.733343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519346579334631:2085] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:02.733378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519342284367304:2085] [node 4] ICP30 unregister session Session# [6:7571519346579334631:2085] VirtualId# [6:7571519346579334626:0]
2025-11-11T17:22:02.733513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [4] session destroyed
2025-11-11T17:22:02.733552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7571519342284367304:2085] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.733568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-11T17:22:02.738004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7571519340638215221:2109] [node 4] ICP33 wake up from error state
2025-11-11T17:22:02.746414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7571519342284367304:2085] [node 4] ICP33 wake up from error state
2025-11-11T17:22:02.861824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [6 <-> 5] connection closed by peer
2025-11-11T17:22:02.862076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7571519342284367324:2079] [node 5] ICS01 socket: 183 reason# EndOfStream
2025-11-11T17:22:02.862121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7571519342284367274:2079] [node 5] ICP30 unregister session Session# [6:7571519342284367324:2079] VirtualId# [6:7571519342284367316:0]
2025-11-11T17:22:02.862164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7571519342284367324:2079] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:02.862270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [5] disconnected
2025-11-11T17:22:02.862328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [5] session destroyed
VERIFY failed (2025-11-11T17:22:03.003809Z): 
  ydb/library/actors/interconnect/ut/dynamic_proxy_ut.cpp:53
  Check(): requirement index == expected failed
NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf<char, std::__y1::char_traits<char>>, char const*, unsigned long)+888 (0xF98C38)
NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+571 (0xF868FB)
TArriveQueue::Check()+2174 (0xB949AE)
RaceTestIter(unsigned int, unsigned int)+7364 (0xB934E4)
NTestSuiteDynamicProxy::TTestCaseRaceCheck10::Execute_(NUnitTest::TTestContext&)+67 (0xB94FA3)
std::__y1::__function::__func<NTestSuiteDynamicProxy::TCurrentTest::Execute()::'lambda'(), std::__y1::allocator<NTestSuiteDynamicProxy::TCurrentTest::Execute()::'lambda'()>, void ()>::operator()()+280 (0xB98FD8)
TColoredProcessor::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+538 (0x1019DDA)
NUnitTest::TTestBase::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+504 (0xFE83B8)
NTestSuiteDynamicProxy::TCurrentTest::Execute()+1229 (0xB9832D)
NUnitTest::TTestFactory::Execute()+2176 (0xFE9B70)
NUnitTest::RunMain(int, char**)+5805 (0x1013B9D)
??+0 (0x7F12F4FF2D90)
__libc_start_main+128 (0x7F12F4FF2E40)
_start+41 (0xAAF029)
2025-11-11T17:22:03.816907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519350653848058:2048] [node 2] ICP01 ready to work
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:03.845864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519350219379036:2048] [node 1] ICP01 ready to work
Subscribe
2025-11-11T17:22:03.885041Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519350653848067:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:03.886452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:03.890105Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519350219379046:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:03.909128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519350653848067:2056] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:03.909493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:03.909712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:03.909824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519350653848070:2048]
2025-11-11T17:22:03.909887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519350653848070:2048] [node 2] ICS09 handshake done sender: [1:7571519350653848067:2056] self: [1:7571519350653848068:0] peer: [2:7571519350219379045:0] socket: 17 qp: 3255
2025-11-11T17:22:03.909933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519350653848070:2048] [node 2] ICS10 traffic start
TEvNodeConnected
2025-11-11T17:22:03.910000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:03.909462Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519350219379046:2056] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:03.909896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:03.909950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:03.909993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519350219379049:2048]
2025-11-11T17:22:03.910025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519350219379049:2048] [node 1] ICS09 handshake done sender: [2:7571519350219379046:2056] self: [2:7571519350219379045:0] peer: [1:7571519350653848068:0] socket: 16 qp: 3256
2025-11-11T17:22:03.910056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519350219379049:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:03.910078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:04.183734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:04.184065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519350653848070:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:04.184092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519350653848070:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:04.184118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:04.184157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519350653848070:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:04.182833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:04.187974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] socket: Connection reset by peer
2025-11-11T17:22:04.188179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519350219379049:2048] [node 1] ICS07 socket disconnect 16 reason# ECONNRESET
2025-11-11T17:22:04.188200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519350219379049:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-11T17:22:04.188232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:04.188302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519350219379049:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:04.199673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519350653848070:2048] [node 2] ICS08 incoming handshake Self# [2:7571519350219379045:0] Peer# [1:7571519350653848068:0] Counter# 38 LastInputSerial# 1551
2025-11-11T17:22:04.199917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519350219379049:2048] [node 1] ICS08 incoming handshake Self# [1:7571519350653848068:0] Peer# [2:7571519350219379045:0] Counter# 1551 LastInputSerial# 38
2025-11-11T17:22:04.202294Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519354948815368:2058] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:04.202574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:04.202670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519350653848070:2048] [node 2] ICS09 handshake done sender: [1:7571519354948815368:2058] self: [1:7571519350653848068:0] peer: [2:7571519350219379045:0] socket: 20 qp: 3259
2025-11-11T17:22:04.203403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519350653848070:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:04.203462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:04.205574Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519354514346348:2058] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:04.210216Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:04.211120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:04.211189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519350219379049:2048] [node 1] ICS09 handshake done sender: [2:7571519354514346348:2058] self: [2:7571519350219379045:0] peer: [1:7571519350653848068:0] socket: 21 qp: 3257
2025-11-11T17:22:04.211923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519350219379049:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:04.211985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:04.212126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:04.530106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:04.530331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519350219379049:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-11-11T17:22:04.530353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519350219379049:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:04.530394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:04.530426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519350219379049:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:04.534998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:04.535198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519350653848070:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:04.535219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519350653848070:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:04.535287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:04.535333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519350653848070:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:04.549398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519350219379049:2048] [node 1] ICS08 incoming handshake Self# [1:7571519350653848068:0] Peer# [2:7571519350219379045:0] Counter# 3484 LastInputSerial# 93
2025-11-11T17:22:04.551513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519350653848070:2048] [node 2] ICS08 incoming handshake Self# [2:7571519350219379045:0] Peer# [1:7571519350653848068:0] Counter# 93 LastInputSerial# 3484
2025-11-11T17:22:04.556482Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519354948815373:2060] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:04.559100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:04.559162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519350653848070:2048] [node 2] ICS09 handshake done sender: [1:7571519354948815373:2060] self: [1:7571519350653848068:0] peer: [2:7571519350219379045:0] socket: 16 qp: 3262
2025-11-11T17:22:04.559649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519350653848070:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:04.559708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:04.567365Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:04.573964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519354514346353:2061] held: no
2025-11-11T17:22:04.574049Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519354514346353:2061] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:04.574348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:04.574417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519350219379049:2048] [node 1] ICS09 handshake done sender: [2:7571519354514346353:2061] self: [2:7571519350219379045:0] peer: [1:7571519350653848068:0] socket: 17 qp: 3261
2025-11-11T17:22:04.575416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519350219379049:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:04.575481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:05.124590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519350219379049:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-11T17:22:05.124660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519350219379049:2048] VirtualId# [2:7571519350219379045:0]
2025-11-11T17:22:05.124734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519350219379049:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:05.124797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:05.124920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:05.129847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:05.130168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519350653848070:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:05.130410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519350653848070:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:05.130483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:05.130528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519350653848070:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:05.152451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519358809313653:2063]) peer tries to resume nonexistent session Self# [1:7571519350653848068:0] Peer# [2:7571519350219379045:0]
2025-11-11T17:22:05.152546Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519358809313653:2063] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:05.153336Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519359243782674:2062] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:05.171750Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:05.171844Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:05.171893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519350653848070:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:05.171956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519350653848070:2048] VirtualId# [1:7571519350653848068:0]
2025-11-11T17:22:05.172029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:05.177419Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519358809313654:2064] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:05.177715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:05.178086Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519359243782677:2063] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:05.189863Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519358809313654:2064] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:05.190164Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519359243782677:2063] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:05.190475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:05.190543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:05.190588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519359243782680:2048]
2025-11-11T17:22:05.190635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519359243782680:2048] [node 2] ICS09 handshake done sender: [1:7571519359243782677:2063] self: [1:7571519359243782676:0] peer: [2:7571519358809313655:0] socket: 16 qp: 3267
2025-11-11T17:22:05.190670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519359243782680:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:05.190707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:05.191609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:05.191673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:05.191725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519358809313657:2048]
2025-11-11T17:22:05.191754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519358809313657:2048] [node 1] ICS09 handshake done sender: [2:7571519358809313654:2064] self: [2:7571519358809313655:0] peer: [1:7571519359243782676:0] socket: 17 qp: 3266
2025-11-11T17:22:05.191777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519358809313657:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:05.191814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:05.529727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:05.530819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:05.531059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519359243782680:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:05.531082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519359243782680:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:05.531115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:05.531160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519359243782680:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:05.533509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:05.533706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519358809313657:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:05.533734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519358809313657:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:05.533819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:05.533857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519358809313657:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:05.535707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519358809313657:2048] [node 1] ICS08 incoming handshake Self# [1:7571519359243782676:0] Peer# [2:7571519358809313655:0] Counter# 1941 LastInputSerial# 51
2025-11-11T17:22:05.543628Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519359243782682:2064] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:05.544721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519359243782680:2048] [node 2] ICS08 incoming handshake Self# [2:7571519358809313655:0] Peer# [1:7571519359243782676:0] Counter# 51 LastInputSerial# 1941
2025-11-11T17:22:05.544763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:05.544812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519359243782680:2048] [node 2] ICS09 handshake done sender: [1:7571519359243782682:2064] self: [1:7571519359243782676:0] peer: [2:7571519358809313655:0] socket: 20 qp: 3268
2025-11-11T17:22:05.545281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519359243782680:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:05.545324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:05.548797Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:05.550059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519358809313660:2067] held: no
2025-11-11T17:22:05.550128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519358809313660:2067] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:05.550399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:05.550446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519358809313657:2048] [node 1] ICS09 handshake done sender: [2:7571519358809313660:2067] self: [2:7571519358809313655:0] peer: [1:7571519359243782676:0] socket: 21 qp: 3269
2025-11-11T17:22:05.551164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519358809313657:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:05.551211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:05.885111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519359243782680:2048] [node 2] ICS01 socket: 20 reason# Debug
2025-11-11T17:22:05.885151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519359243782680:2048] VirtualId# [1:7571519359243782676:0]
2025-11-11T17:22:05.885178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519359243782680:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:05.885219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:05.885270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:05.886482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:05.886671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519358809313657:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-11T17:22:05.886691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519358809313657:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:05.886746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:05.886775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519358809313657:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:05.890725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519359243782687:2066]) peer tries to resume nonexistent session Self# [2:7571519358809313655:0] Peer# [1:7571519359243782676:0]
2025-11-11T17:22:05.890808Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519359243782687:2066] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:05.891514Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519358809313664:2069] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:05.892504Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:05.892562Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:05.892586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519358809313657:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:05.892614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519358809313657:2048] VirtualId# [2:7571519358809313655:0]
2025-11-11T17:22:05.892657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:05.903419Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519358809313666:2070] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:05.905084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:05.910055Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519359243782689:2067] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:05.910300Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519359243782691:2068] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:05.916107Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519358809313666:2070] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:05.918010Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519359243782689:2067] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:05.918317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:05.918424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:05.918461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519359243782694:2048]
2025-11-11T17:22:05.918495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519359243782694:2048] [node 2] ICS09 handshake done sender: [1:7571519359243782689:2067] self: [1:7571519359243782688:0] peer: [2:7571519358809313670:0] socket: 18 qp: 3273
2025-11-11T17:22:05.918523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519359243782694:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:05.918554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:05.921901Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:05.925319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:05.925373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:05.925409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519358809313672:2048]
2025-11-11T17:22:05.925443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519358809313672:2048] [node 1] ICS09 handshake done sender: [2:7571519358809313666:2070] self: [2:7571519358809313670:0] peer: [1:7571519359243782688:0] socket: 19 qp: 3274
2025-11-11T17:22:05.925469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519358809313672:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:05.925496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:05.925606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:06.174720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519358809313672:2048] [node 1] ICS01 socket: 19 reason# Debug
2025-11-11T17:22:06.174765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519358809313672:2048] VirtualId# [2:7571519358809313670:0]
2025-11-11T17:22:06.174791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519358809313672:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:06.174841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:06.174924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:06.175937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:06.176176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519359243782694:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:06.176198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519359243782694:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:06.176259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:06.176292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519359243782694:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 2931
TEvUndelivered Cookie# 2932
TEvUndelivered Cookie# 2933
TEvUndelivered Cookie# 2934
TEvUndelivered Cookie# 2935
TEvUndelivered Cookie# 2936
TEvUndelivered Cookie# 2937
TEvUndelivered Cookie# 2938
2025-11-11T17:22:06.184415Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519363538749992:2069] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:06.184127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519363104280970:2073]) peer tries to resume nonexistent session Self# [1:7571519359243782688:0] Peer# [2:7571519358809313670:0]
2025-11-11T17:22:06.184195Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519363104280970:2073] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:06.186306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:06.186366Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:06.186389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519359243782694:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:06.186414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519359243782694:2048] VirtualId# [1:7571519359243782688:0]
2025-11-11T17:22:06.186449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:06.190671Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519363538749994:2070] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:06.192746Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519363104280974:2075] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:06.192837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:06.193160Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519363538749996:2071] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:06.201934Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519363104280974:2075] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:06.202157Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519363538749996:2071] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:06.202482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:06.202534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:06.202571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519363538750000:2048]
2025-11-11T17:22:06.202602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519363538750000:2048] [node 2] ICS09 handshake done sender: [1:7571519363538749996:2071] self: [1:7571519363538749995:0] peer: [2:7571519363104280975:0] socket: 16 qp: 3281
2025-11-11T17:22:06.202630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519363538750000:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:06.202671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:06.204052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:06.204094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:06.204128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519363104280977:2048]
2025-11-11T17:22:06.204157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519363104280977:2048] [node 1] ICS09 handshake done sender: [2:7571519363104280974:2075] self: [2:7571519363104280975:0] peer: [1:7571519363538749995:0] socket: 17 qp: 3280
2025-11-11T17:22:06.204180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519363104280977:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:06.204204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:06.204646Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:06.207133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:06.434988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:06.435177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:06.435203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519363104280977:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:06.435230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519363104280977:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:06.435258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:06.435649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:06.435892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519363538750000:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:06.435914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519363538750000:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:06.435966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:06.436008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519363538750000:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:06.438282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:06.438466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519363104280977:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:06.438492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519363104280977:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:06.440423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519363104280977:2048] [node 1] ICS08 incoming handshake Self# [1:7571519363538749995:0] Peer# [2:7571519363104280975:0] Counter# 1369 LastInputSerial# 35
2025-11-11T17:22:06.442543Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519363538750002:2072] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:06.442815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:06.442872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519363538750000:2048] [node 2] ICS09 handshake done sender: [1:7571519363538750002:2072] self: [1:7571519363538749995:0] peer: [2:7571519363104280975:0] socket: 19 qp: 3282
2025-11-11T17:22:06.443385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519363538750000:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:06.443438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:06.447368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519363538750000:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:06.447405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519363538750000:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:06.447455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:06.447493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519363538750000:2048] [node 2] ICS08 incoming handshake Self# [2:7571519363104280975:0] Peer# [1:7571519363538749995:0] Counter# 35 LastInputSerial# 1369
2025-11-11T17:22:06.447616Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:06.448913Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519363104280980:2078] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:06.447798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519363538750000:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:06.447841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519363538750000:2048] VirtualId# [1:7571519363538749995:0]
2025-11-11T17:22:06.447882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:06.450297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:06.450353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519363104280977:2048] [node 1] ICS09 handshake done sender: [2:7571519363104280980:2078] self: [2:7571519363104280975:0] peer: [1:7571519363538749995:0] socket: 21 qp: 3283
2025-11-11T17:22:06.451021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519363104280977:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:06.451061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:06.451368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:06.451390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519363104280977:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:06.451433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519363104280977:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:06.451468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:06.451549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:06.451708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519363104280977:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:06.451725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519363104280977:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:06.455309Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:06.458677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519363538750008:2075]) peer tries to resume nonexistent session Self# [2:7571519363104280975:0] Peer# [1:7571519363538749995:0]
2025-11-11T17:22:06.458729Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519363538750008:2075] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:06.459877Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519363104280985:2080] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:06.461053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:06.461123Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:06.461168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519363104280977:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:06.461195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519363104280977:2048] VirtualId# [2:7571519363104280975:0]
2025-11-11T17:22:06.461297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 3282
TEvUndelivered Cookie# 3283
TEvUndelivered Cookie# 3284
TEvUndelivered Cookie# 3285
TEvUndelivered Cookie# 3286
2025-11-11T17:22:06.466778Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519363538750009:2076] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:06.466885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:06.467955Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519363104280988:2081] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:06.471774Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519363104280988:2081] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:06.472080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:06.472135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:06.472174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519363104280991:2048]
2025-11-11T17:22:06.472209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519363104280991:2048] [node 1] ICS09 handshake done sender: [2:7571519363104280988:2081] self: [2:7571519363104280987:0] peer: [1:7571519363538750010:0] socket: 16 qp: 3288
2025-11-11T17:22:06.472235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519363104280991:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:06.472267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:06.471650Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519363538750009:2076] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:06.472917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:06.472946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:06.472976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519363538750012:2048]
2025-11-11T17:22:06.473009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519363538750012:2048] [node 2] ICS09 handshake done sender: [1:7571519363538750009:2076] self: [1:7571519363538750010:0] peer: [2:7571519363104280987:0] socket: 17 qp: 3289
2025-11-11T17:22:06.473029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519363538750012:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:06.473049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:06.815262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:06.815505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519363104280991:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:06.815527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519363104280991:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:06.815569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:06.815594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519363104280991:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:06.818501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:06.818736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519363538750012:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:06.818755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519363538750012:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:06.818802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:06.818833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519363538750012:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:06.826459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519363104280991:2048] [node 1] ICS08 incoming handshake Self# [1:7571519363538750010:0] Peer# [2:7571519363104280987:0] Counter# 2080 LastInputSerial# 55
2025-11-11T17:22:06.826752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519363538750012:2048] [node 2] ICS08 incoming handshake Self# [2:7571519363104280987:0] Peer# [1:7571519363538750010:0] Counter# 55 LastInputSerial# 2080
2025-11-11T17:22:06.827138Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519363538750014:2078] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:06.827433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:06.827481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519363538750012:2048] [node 2] ICS09 handshake done sender: [1:7571519363538750014:2078] self: [1:7571519363538750010:0] peer: [2:7571519363104280987:0] socket: 22 qp: 3291
2025-11-11T17:22:06.827942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519363538750012:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:06.827982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:06.831810Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519363104280995:2083] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:06.832545Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:06.833472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:06.833525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519363104280991:2048] [node 1] ICS09 handshake done sender: [2:7571519363104280995:2083] self: [2:7571519363104280987:0] peer: [1:7571519363538750010:0] socket: 23 qp: 3293
2025-11-11T17:22:06.834215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519363104280991:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:06.834279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:06.834425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:07.259503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:07.259708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519363104280991:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-11T17:22:07.259730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519363104280991:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:07.259788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:07.259819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519363104280991:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:07.258077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519363538750012:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-11T17:22:07.258125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519363538750012:2048] VirtualId# [1:7571519363538750010:0]
2025-11-11T17:22:07.258155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519363538750012:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:07.258195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:07.258250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:07.270253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519367833717315:2080]) peer tries to resume nonexistent session Self# [2:7571519363104280987:0] Peer# [1:7571519363538750010:0]
2025-11-11T17:22:07.270301Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519367833717315:2080] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:07.270965Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519367399248296:2086] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:07.271034Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519367399248294:2085] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:07.272143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:07.272208Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:07.272230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519363104280991:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:07.272257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519363104280991:2048] VirtualId# [2:7571519363104280987:0]
2025-11-11T17:22:07.272302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:07.273399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:07.273996Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519367833717317:2081] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:07.279774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519367833717319:2082] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:07.282128Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519367399248296:2086] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:07.282465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:07.282512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:07.282549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519367399248302:2048]
2025-11-11T17:22:07.282584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519367399248302:2048] [node 1] ICS09 handshake done sender: [2:7571519367399248296:2086] self: [2:7571519367399248299:0] peer: [1:7571519367833717316:0] socket: 19 qp: 3297
2025-11-11T17:22:07.282619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519367399248302:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:07.282645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:07.283849Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519367833717317:2081] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:07.288982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:07.289042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:07.289082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519367833717322:2048]
2025-11-11T17:22:07.289115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519367833717322:2048] [node 2] ICS09 handshake done sender: [1:7571519367833717317:2081] self: [1:7571519367833717316:0] peer: [2:7571519367399248299:0] socket: 18 qp: 3296
2025-11-11T17:22:07.289142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519367833717322:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:07.289172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:07.741744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:07.741974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519367833717322:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:07.741998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519367833717322:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:07.742048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:07.742078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519367833717322:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:07.743042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:07.743074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519367399248302:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:07.743096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519367399248302:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:07.743116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:07.744930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:07.745113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519367399248302:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:07.745129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519367399248302:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:07.748643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519367399248302:2048] [node 1] ICS08 incoming handshake Self# [1:7571519367833717316:0] Peer# [2:7571519367399248299:0] Counter# 3375 LastInputSerial# 88
2025-11-11T17:22:07.749195Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519367833717324:2083] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:07.749498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:07.749545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519367833717322:2048] [node 2] ICS09 handshake done sender: [1:7571519367833717324:2083] self: [1:7571519367833717316:0] peer: [2:7571519367399248299:0] socket: 16 qp: 3300
2025-11-11T17:22:07.749951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519367833717322:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:07.749988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:07.753211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519367833717322:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:07.753239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519367833717322:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:07.753281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:07.753317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519367833717322:2048] [node 2] ICS08 incoming handshake Self# [2:7571519367399248299:0] Peer# [1:7571519367833717316:0] Counter# 88 LastInputSerial# 3375
2025-11-11T17:22:07.753549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:07.753731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519367833717322:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:07.753757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519367833717322:2048] VirtualId# [1:7571519367833717316:0]
2025-11-11T17:22:07.753789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:07.755459Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519367399248305:2090] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:07.755801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:07.755847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519367399248302:2048] [node 1] ICS09 handshake done sender: [2:7571519367399248305:2090] self: [2:7571519367399248299:0] peer: [1:7571519367833717316:0] socket: 17 qp: 3302
2025-11-11T17:22:07.756358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519367399248302:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:07.756393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:07.756673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:07.756693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519367399248302:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:07.756713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519367399248302:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:07.756729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:07.756789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:07.756922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519367399248302:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:07.756937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519367399248302:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:07.759681Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:07.763489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519367833717330:2086]) peer tries to resume nonexistent session Self# [2:7571519367399248299:0] Peer# [1:7571519367833717316:0]
2025-11-11T17:22:07.763538Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519367833717330:2086] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:07.764249Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519367399248310:2092] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:07.765141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:07.765203Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:07.765225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519367399248302:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:07.765250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519367399248302:2048] VirtualId# [2:7571519367399248299:0]
2025-11-11T17:22:07.765333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 5522
TEvUndelivered Cookie# 5523
TEvUndelivered Cookie# 5524
TEvUndelivered Cookie# 5525
TEvUndelivered Cookie# 5526
2025-11-11T17:22:07.769781Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519367833717331:2087] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:07.769895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:07.773967Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519367399248313:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:07.781818Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519367833717331:2087] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:07.782068Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519367399248313:2093] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:07.782339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:07.782382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:07.782417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519367399248316:2048]
2025-11-11T17:22:07.782451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519367399248316:2048] [node 1] ICS09 handshake done sender: [2:7571519367399248313:2093] self: [2:7571519367399248312:0] peer: [1:7571519367833717332:0] socket: 16 qp: 3306
2025-11-11T17:22:07.782479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519367399248316:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:07.782509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:07.782696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:07.782736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:07.782771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519367833717334:2048]
2025-11-11T17:22:07.782802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519367833717334:2048] [node 2] ICS09 handshake done sender: [1:7571519367833717331:2087] self: [1:7571519367833717332:0] peer: [2:7571519367399248312:0] socket: 17 qp: 3305
2025-11-11T17:22:07.782834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519367833717334:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:07.782856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:08.009884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519367399248316:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:08.009934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519367399248316:2048] VirtualId# [2:7571519367399248312:0]
2025-11-11T17:22:08.009965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519367399248316:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:08.010023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:08.010070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:08.014051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:08.014273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519367833717334:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:08.014296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519367833717334:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:08.014378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:08.014412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519367833717334:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:08.025244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519371694215614:2094]) peer tries to resume nonexistent session Self# [1:7571519367833717332:0] Peer# [2:7571519367399248312:0]
2025-11-11T17:22:08.025325Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519371694215614:2094] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:08.025865Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519372128684632:2089] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:08.027505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:08.027572Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:08.027594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519367833717334:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:08.027623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519367833717334:2048] VirtualId# [1:7571519367833717332:0]
2025-11-11T17:22:08.027667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:08.041820Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519371694215618:2096] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:08.042100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:08.045801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519372128684637:2091] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:08.045915Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519372128684635:2090] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:08.065831Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519371694215618:2096] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:08.069856Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519372128684635:2090] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:08.070192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:08.070246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:08.070283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519372128684640:2048]
2025-11-11T17:22:08.070317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519372128684640:2048] [node 2] ICS09 handshake done sender: [1:7571519372128684635:2090] self: [1:7571519372128684634:0] peer: [2:7571519371694215619:0] socket: 17 qp: 3309
2025-11-11T17:22:08.070353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519372128684640:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:08.070379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:08.073896Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:08.081945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:08.082005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:08.082049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519371694215621:2048]
2025-11-11T17:22:08.082085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519371694215621:2048] [node 1] ICS09 handshake done sender: [2:7571519371694215618:2096] self: [2:7571519371694215619:0] peer: [1:7571519372128684634:0] socket: 18 qp: 3308
2025-11-11T17:22:08.082116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519371694215621:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:08.082173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:08.082378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:08.321972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:08.322163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519371694215621:2048] [node 1] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:08.322186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519371694215621:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:08.322254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:08.322282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519371694215621:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:08.324767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:08.324989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519372128684640:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:08.325012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519372128684640:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:08.325062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:08.325094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519372128684640:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:08.338330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519372128684640:2048] [node 2] ICS08 incoming handshake Self# [2:7571519371694215619:0] Peer# [1:7571519372128684634:0] Counter# 24 LastInputSerial# 997
2025-11-11T17:22:08.341822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519371694215621:2048] [node 1] ICS08 incoming handshake Self# [1:7571519372128684634:0] Peer# [2:7571519371694215619:0] Counter# 997 LastInputSerial# 24
2025-11-11T17:22:08.353843Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519372128684642:2092] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:08.362564Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519371694215625:2099] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:08.365835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:08.365898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519372128684640:2048] [node 2] ICS09 handshake done sender: [1:7571519372128684642:2092] self: [1:7571519372128684634:0] peer: [2:7571519371694215619:0] socket: 22 qp: 3314
2025-11-11T17:22:08.366281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519372128684640:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:08.366321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:08.369864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:08.369923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519371694215621:2048] [node 1] ICS09 handshake done sender: [2:7571519371694215625:2099] self: [2:7571519371694215619:0] peer: [1:7571519372128684634:0] socket: 23 qp: 3315
2025-11-11T17:22:08.370455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519371694215621:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:08.370504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:08.558769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:08.558897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:08.559069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519371694215621:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-11T17:22:08.559092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519371694215621:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:08.559114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:08.559143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519371694215621:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:08.561819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:08.562023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519372128684640:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:08.562045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519372128684640:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:08.562093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:08.562122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519372128684640:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:08.569021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519371694215621:2048] [node 1] ICS08 incoming handshake Self# [1:7571519372128684634:0] Peer# [2:7571519371694215619:0] Counter# 2626 LastInputSerial# 64
2025-11-11T17:22:08.569357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519372128684640:2048] [node 2] ICS08 incoming handshake Self# [2:7571519371694215619:0] Peer# [1:7571519372128684634:0] Counter# 64 LastInputSerial# 2626
2025-11-11T17:22:08.569746Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519372128684647:2094] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:08.571509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:08.571566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519372128684640:2048] [node 2] ICS09 handshake done sender: [1:7571519372128684647:2094] self: [1:7571519372128684634:0] peer: [2:7571519371694215619:0] socket: 18 qp: 3317
2025-11-11T17:22:08.572044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519372128684640:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:08.572083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:08.574200Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:08.575114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519371694215630:2102] held: no
2025-11-11T17:22:08.575175Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519371694215630:2102] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:08.575386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:08.575429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519371694215621:2048] [node 1] ICS09 handshake done sender: [2:7571519371694215630:2102] self: [2:7571519371694215619:0] peer: [1:7571519372128684634:0] socket: 19 qp: 3319
2025-11-11T17:22:08.576050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519371694215621:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:08.576084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:08.714413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519371694215621:2048] [node 1] ICS01 socket: 19 reason# Debug
2025-11-11T17:22:08.714458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519371694215621:2048] VirtualId# [2:7571519371694215619:0]
2025-11-11T17:22:08.714488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519371694215621:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:08.714535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:08.714576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:08.715762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:08.715973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519372128684640:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:08.715993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519372128684640:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:08.716047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:08.716075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519372128684640:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:08.719956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519371694215633:2104]) peer tries to resume nonexistent session Self# [1:7571519372128684634:0] Peer# [2:7571519371694215619:0]
2025-11-11T17:22:08.720015Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519371694215633:2104] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:08.720822Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519372128684652:2096] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:08.721804Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:08.721870Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:08.721892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519372128684640:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:08.721917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519372128684640:2048] VirtualId# [1:7571519372128684634:0]
2025-11-11T17:22:08.721961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:08.726801Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519371694215634:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:08.726936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:08.727288Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519372128684655:2097] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:08.732388Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519371694215634:2105] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:08.732829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:08.732880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:08.732920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519371694215637:2048]
2025-11-11T17:22:08.732961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519371694215637:2048] [node 1] ICS09 handshake done sender: [2:7571519371694215634:2105] self: [2:7571519371694215635:0] peer: [1:7571519372128684654:0] socket: 17 qp: 3321
2025-11-11T17:22:08.732997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519371694215637:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:08.733034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:08.733709Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519372128684655:2097] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:08.734006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:08.734040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:08.734073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519372128684658:2048]
2025-11-11T17:22:08.734104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519372128684658:2048] [node 2] ICS09 handshake done sender: [1:7571519372128684655:2097] self: [1:7571519372128684654:0] peer: [2:7571519371694215635:0] socket: 16 qp: 3322
2025-11-11T17:22:08.734129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519372128684658:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:08.734152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:09.079897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:09.080205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519372128684658:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:09.080235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519372128684658:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:09.080282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:09.080310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519372128684658:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:09.081865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:09.082066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519371694215637:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:09.082088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519371694215637:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:09.082139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:09.082169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519371694215637:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:09.087593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519371694215637:2048] [node 1] ICS08 incoming handshake Self# [1:7571519372128684654:0] Peer# [2:7571519371694215635:0] Counter# 1462 LastInputSerial# 35
2025-11-11T17:22:09.089310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519372128684658:2048] [node 2] ICS08 incoming handshake Self# [2:7571519371694215635:0] Peer# [1:7571519372128684654:0] Counter# 35 LastInputSerial# 1462
2025-11-11T17:22:09.089802Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519376423651956:2098] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:09.090152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:09.090208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519372128684658:2048] [node 2] ICS09 handshake done sender: [1:7571519376423651956:2098] self: [1:7571519372128684654:0] peer: [2:7571519371694215635:0] socket: 20 qp: 3324
2025-11-11T17:22:09.090968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519372128684658:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:09.091013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:09.094486Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519375989182936:2108] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:09.095092Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:09.095979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:09.096035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519371694215637:2048] [node 1] ICS09 handshake done sender: [2:7571519375989182936:2108] self: [2:7571519371694215635:0] peer: [1:7571519372128684654:0] socket: 21 qp: 3323
2025-11-11T17:22:09.096593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519371694215637:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:09.096631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:09.096739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:09.647002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519371694215637:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-11-11T17:22:09.647046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519371694215637:2048] VirtualId# [2:7571519371694215635:0]
2025-11-11T17:22:09.647077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519371694215637:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:09.647128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:09.647171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:09.648148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:09.648365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519372128684658:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:09.648388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519372128684658:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:09.648445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:09.648473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519372128684658:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:09.652409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519375989182940:2110]) peer tries to resume nonexistent session Self# [1:7571519372128684654:0] Peer# [2:7571519371694215635:0]
2025-11-11T17:22:09.652475Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519375989182940:2110] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:09.653119Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519376423651961:2100] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:09.654137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:09.654208Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:09.654234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519372128684658:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:09.654262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519372128684658:2048] VirtualId# [1:7571519372128684654:0]
2025-11-11T17:22:09.654309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:09.659419Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519375989182941:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:09.659574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:09.659994Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519376423651964:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:09.669858Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519375989182941:2111] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:09.673999Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519376423651964:2101] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:09.674220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:09.674269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:09.674314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519375989182944:2048]
2025-11-11T17:22:09.674352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519375989182944:2048] [node 1] ICS09 handshake done sender: [2:7571519375989182941:2111] self: [2:7571519375989182942:0] peer: [1:7571519376423651963:0] socket: 17 qp: 3328
2025-11-11T17:22:09.674383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519375989182944:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:09.674416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:09.675083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:09.675121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:09.675152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519376423651967:2048]
2025-11-11T17:22:09.675181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519376423651967:2048] [node 2] ICS09 handshake done sender: [1:7571519376423651964:2101] self: [1:7571519376423651963:0] peer: [2:7571519375989182942:0] socket: 16 qp: 3329
2025-11-11T17:22:09.675204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519376423651967:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:09.675226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:09.986568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:09.986851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519375989182944:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-11T17:22:09.986873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519375989182944:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:09.986913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:09.986942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519375989182944:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:09.987686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:09.987888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519376423651967:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:09.987909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519376423651967:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:09.987955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:09.987983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519376423651967:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:09.992400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519376423651967:2048] [node 2] ICS08 incoming handshake Self# [2:7571519375989182942:0] Peer# [1:7571519376423651963:0] Counter# 46 LastInputSerial# 1818
2025-11-11T17:22:09.992999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519375989182944:2048] [node 1] ICS08 incoming handshake Self# [1:7571519376423651963:0] Peer# [2:7571519375989182942:0] Counter# 1818 LastInputSerial# 46
2025-11-11T17:22:09.993806Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519376423651969:2102] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:09.994093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:09.994154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519376423651967:2048] [node 2] ICS09 handshake done sender: [1:7571519376423651969:2102] self: [1:7571519376423651963:0] peer: [2:7571519375989182942:0] socket: 22 qp: 3332
2025-11-11T17:22:09.994659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519376423651967:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:09.994695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:09.998743Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:09.999816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519375989182948:2114] held: no
2025-11-11T17:22:09.999880Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519375989182948:2114] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:10.000139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:10.000184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519375989182944:2048] [node 1] ICS09 handshake done sender: [2:7571519375989182948:2114] self: [2:7571519375989182942:0] peer: [1:7571519376423651963:0] socket: 23 qp: 3333
2025-11-11T17:22:10.000840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519375989182944:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:10.000892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:10.239623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:10.240982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519375989182944:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-11T17:22:10.241010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519375989182944:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:10.241093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:10.241123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519375989182944:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:10.241876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:10.242113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519376423651967:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:10.242141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519376423651967:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:10.242192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:10.242220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519376423651967:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:10.254636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519376423651967:2048] [node 2] ICS08 incoming handshake Self# [2:7571519375989182942:0] Peer# [1:7571519376423651963:0] Counter# 98 LastInputSerial# 3532
2025-11-11T17:22:10.254848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519375989182944:2048] [node 1] ICS08 incoming handshake Self# [1:7571519376423651963:0] Peer# [2:7571519375989182942:0] Counter# 3532 LastInputSerial# 98
2025-11-11T17:22:10.255297Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519380718619270:2104] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:10.255552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:10.255604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519376423651967:2048] [node 2] ICS09 handshake done sender: [1:7571519380718619270:2104] self: [1:7571519376423651963:0] peer: [2:7571519375989182942:0] socket: 18 qp: 3335
2025-11-11T17:22:10.256015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519376423651967:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:10.256050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:10.259616Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:10.260550Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519380284150249:2117] held: no
2025-11-11T17:22:10.260612Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519380284150249:2117] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:10.260832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:10.260888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519375989182944:2048] [node 1] ICS09 handshake done sender: [2:7571519380284150249:2117] self: [2:7571519375989182942:0] peer: [1:7571519376423651963:0] socket: 19 qp: 3337
2025-11-11T17:22:10.261392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519375989182944:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:10.261429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:10.825896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:10.826668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:10.826906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519376423651967:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:10.826929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519376423651967:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:10.826984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:10.827012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519376423651967:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:10.828485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:10.828654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519375989182944:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:10.828676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519375989182944:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:10.828699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:10.828739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519375989182944:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:10.834498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519375989182944:2048] [node 1] ICS08 incoming handshake Self# [1:7571519376423651963:0] Peer# [2:7571519375989182942:0] Counter# 8048 LastInputSerial# 215
2025-11-11T17:22:10.836127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519376423651967:2048] [node 2] ICS08 incoming handshake Self# [2:7571519375989182942:0] Peer# [1:7571519376423651963:0] Counter# 215 LastInputSerial# 8048
2025-11-11T17:22:10.836591Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519380718619275:2106] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:10.836986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:10.837035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519376423651967:2048] [node 2] ICS09 handshake done sender: [1:7571519380718619275:2106] self: [1:7571519376423651963:0] peer: [2:7571519375989182942:0] socket: 16 qp: 3339
2025-11-11T17:22:10.837535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519376423651967:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:10.837584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:10.839546Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519380284150253:2120] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:10.839844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:10.839891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519375989182944:2048] [node 1] ICS09 handshake done sender: [2:7571519380284150253:2120] self: [2:7571519375989182942:0] peer: [1:7571519376423651963:0] socket: 17 qp: 3338
2025-11-11T17:22:10.840397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519375989182944:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:10.840445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:11.198441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:11.199791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519376423651967:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:11.199824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519376423651967:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:11.199858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:11.199898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519376423651967:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:11.201878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:11.202060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519375989182944:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:11.202083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519375989182944:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:11.202140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:11.202170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519375989182944:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:11.205225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519375989182944:2048] [node 1] ICS08 incoming handshake Self# [1:7571519376423651963:0] Peer# [2:7571519375989182942:0] Counter# 11683 LastInputSerial# 307
2025-11-11T17:22:11.207403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519376423651967:2048] [node 2] ICS08 incoming handshake Self# [2:7571519375989182942:0] Peer# [1:7571519376423651963:0] Counter# 307 LastInputSerial# 11683
2025-11-11T17:22:11.207761Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519385013586576:2108] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:11.208017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:11.208064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519376423651967:2048] [node 2] ICS09 handshake done sender: [1:7571519385013586576:2108] self: [1:7571519376423651963:0] peer: [2:7571519375989182942:0] socket: 18 qp: 3342
2025-11-11T17:22:11.208420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519376423651967:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:11.208453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:11.211658Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:11.212592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519384579117554:2123] held: no
2025-11-11T17:22:11.212653Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519384579117554:2123] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:11.212851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:11.212893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519375989182944:2048] [node 1] ICS09 handshake done sender: [2:7571519384579117554:2123] self: [2:7571519375989182942:0] peer: [1:7571519376423651963:0] socket: 19 qp: 3343
2025-11-11T17:22:11.213489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519375989182944:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:11.213527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:11.797184Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:11.797425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519376423651967:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:11.797450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519376423651967:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:11.797486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:11.797515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519376423651967:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:11.799474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:11.799660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519375989182944:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:11.799679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519375989182944:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:11.799730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:11.799757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519375989182944:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:11.804780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519375989182944:2048] [node 1] ICS08 incoming handshake Self# [1:7571519376423651963:0] Peer# [2:7571519375989182942:0] Counter# 17451 LastInputSerial# 453
2025-11-11T17:22:11.806368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519376423651967:2048] [node 2] ICS08 incoming handshake Self# [2:7571519375989182942:0] Peer# [1:7571519376423651963:0] Counter# 453 LastInputSerial# 17451
2025-11-11T17:22:11.808175Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519385013586581:2110] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:11.808438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:11.808488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519376423651967:2048] [node 2] ICS09 handshake done sender: [1:7571519385013586581:2110] self: [1:7571519376423651963:0] peer: [2:7571519375989182942:0] socket: 16 qp: 3347
2025-11-11T17:22:11.808963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519376423651967:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:11.809000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:11.811661Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519384579117559:2126] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:11.812238Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:11.813091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:11.813141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519375989182944:2048] [node 1] ICS09 handshake done sender: [2:7571519384579117559:2126] self: [2:7571519375989182942:0] peer: [1:7571519376423651963:0] socket: 17 qp: 3346
2025-11-11T17:22:11.813623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519375989182944:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:11.813662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:11.813776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:12.217425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:12.217653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519376423651967:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:12.217683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519376423651967:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:12.217733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:12.217766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519376423651967:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:12.227847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:12.228074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519375989182944:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:12.228099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519375989182944:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:12.228146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:12.228180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519375989182944:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:12.233848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519375989182944:2048] [node 1] ICS08 incoming handshake Self# [1:7571519376423651963:0] Peer# [2:7571519375989182942:0] Counter# 19909 LastInputSerial# 516
2025-11-11T17:22:12.234470Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519389308553882:2112] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:12.234732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:12.234780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519376423651967:2048] [node 2] ICS09 handshake done sender: [1:7571519389308553882:2112] self: [1:7571519376423651963:0] peer: [2:7571519375989182942:0] socket: 18 qp: 3350
2025-11-11T17:22:12.235220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519376423651967:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:12.235259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:12.238975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519376423651967:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:12.239004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519376423651967:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:12.239055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:12.239095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519376423651967:2048] [node 2] ICS08 incoming handshake Self# [2:7571519375989182942:0] Peer# [1:7571519376423651963:0] Counter# 516 LastInputSerial# 19909
2025-11-11T17:22:12.239386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:12.239583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519376423651967:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:12.239612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519376423651967:2048] VirtualId# [1:7571519376423651963:0]
2025-11-11T17:22:12.239642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:12.241343Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519388874084860:2129] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:12.241748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:12.241803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519375989182944:2048] [node 1] ICS09 handshake done sender: [2:7571519388874084860:2129] self: [2:7571519375989182942:0] peer: [1:7571519376423651963:0] socket: 19 qp: 3351
2025-11-11T17:22:12.242413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519375989182944:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:12.242453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:12.242738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:12.242758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519375989182944:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:12.242777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519375989182944:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:12.242797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:12.242853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:12.243015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519375989182944:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:12.243031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519375989182944:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:12.246067Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:12.250327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519389308553888:2115]) peer tries to resume nonexistent session Self# [2:7571519375989182942:0] Peer# [1:7571519376423651963:0]
2025-11-11T17:22:12.250370Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519389308553888:2115] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:12.251205Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519388874084865:2131] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:12.252186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:12.252249Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:12.252271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519375989182944:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:12.252299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519375989182944:2048] VirtualId# [2:7571519375989182942:0]
2025-11-11T17:22:12.252419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 13151
TEvUndelivered Cookie# 13152
TEvUndelivered Cookie# 13153
TEvUndelivered Cookie# 13154
TEvUndelivered Cookie# 13155
TEvUndelivered Cookie# 13156
TEvUndelivered Cookie# 13157
TEvUndelivered Cookie# 13158
TEvUndelivered Cookie# 13159
2025-11-11T17:22:12.257107Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519389308553889:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:12.257228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:12.258385Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519388874084868:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:12.269982Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519389308553889:2116] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:12.274176Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519388874084868:2132] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:12.274047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:12.274100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:12.274149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519389308553892:2048]
2025-11-11T17:22:12.274191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519389308553892:2048] [node 2] ICS09 handshake done sender: [1:7571519389308553889:2116] self: [1:7571519389308553890:0] peer: [2:7571519388874084867:0] socket: 17 qp: 3355
2025-11-11T17:22:12.274222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519389308553892:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:12.274272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:12.274654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:12.274693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:12.274730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519388874084871:2048]
2025-11-11T17:22:12.274767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519388874084871:2048] [node 1] ICS09 handshake done sender: [2:7571519388874084868:2132] self: [2:7571519388874084867:0] peer: [1:7571519389308553890:0] socket: 16 qp: 3356
2025-11-11T17:22:12.274791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519388874084871:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:12.274816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:12.732728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:12.732979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519388874084871:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:12.733006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519388874084871:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:12.733053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:12.733079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519388874084871:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:12.735807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:12.736043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519389308553892:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:12.736066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519389308553892:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:12.736124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:12.736155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519389308553892:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:12.743093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519388874084871:2048] [node 1] ICS08 incoming handshake Self# [1:7571519389308553890:0] Peer# [2:7571519388874084867:0] Counter# 3120 LastInputSerial# 75
2025-11-11T17:22:12.744153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519389308553892:2048] [node 2] ICS08 incoming handshake Self# [2:7571519388874084867:0] Peer# [1:7571519389308553890:0] Counter# 75 LastInputSerial# 3120
2025-11-11T17:22:12.744521Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519389308553894:2118] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:12.744761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:12.744827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519389308553892:2048] [node 2] ICS09 handshake done sender: [1:7571519389308553894:2118] self: [1:7571519389308553890:0] peer: [2:7571519388874084867:0] socket: 22 qp: 3358
2025-11-11T17:22:12.745325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519389308553892:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:12.745363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:12.746782Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:12.747894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519388874084875:2134] held: no
2025-11-11T17:22:12.749862Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519388874084875:2134] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:12.750034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:12.750083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519388874084871:2048] [node 1] ICS09 handshake done sender: [2:7571519388874084875:2134] self: [2:7571519388874084867:0] peer: [1:7571519389308553890:0] socket: 23 qp: 3360
2025-11-11T17:22:12.750866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519388874084871:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:12.750907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:13.238033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:13.238340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519388874084871:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-11T17:22:13.238366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519388874084871:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:13.238408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:13.238440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519388874084871:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:13.240145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:13.240376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519389308553892:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:13.240398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519389308553892:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:13.240446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:13.240478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519389308553892:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:13.247071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519388874084871:2048] [node 1] ICS08 incoming handshake Self# [1:7571519389308553890:0] Peer# [2:7571519388874084867:0] Counter# 6433 LastInputSerial# 158
2025-11-11T17:22:13.250741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519389308553892:2048] [node 2] ICS08 incoming handshake Self# [2:7571519388874084867:0] Peer# [1:7571519389308553890:0] Counter# 158 LastInputSerial# 6433
2025-11-11T17:22:13.252098Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519393603521195:2120] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:13.252357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:13.252411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519389308553892:2048] [node 2] ICS09 handshake done sender: [1:7571519393603521195:2120] self: [1:7571519389308553890:0] peer: [2:7571519388874084867:0] socket: 19 qp: 3362
2025-11-11T17:22:13.252883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519389308553892:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:13.252922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:13.264588Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:13.268115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519393169052176:2137] held: no
2025-11-11T17:22:13.268195Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519393169052176:2137] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:13.268461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:13.268511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519388874084871:2048] [node 1] ICS09 handshake done sender: [2:7571519393169052176:2137] self: [2:7571519388874084867:0] peer: [1:7571519389308553890:0] socket: 20 qp: 3364
2025-11-11T17:22:13.269077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519388874084871:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:13.269120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:13.675552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519388874084871:2048] [node 1] ICS01 socket: 20 reason# Debug
2025-11-11T17:22:13.675597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519388874084871:2048] VirtualId# [2:7571519388874084867:0]
2025-11-11T17:22:13.675626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519388874084871:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:13.675669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:13.675714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:13.676897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:13.680149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519389308553892:2048] [node 2] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:13.680180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519389308553892:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
TEvNodeDisconnected
2025-11-11T17:22:13.683292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:13.683363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519389308553892:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:13.687325Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519393603521201:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:13.688668Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:13.688700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519389308553892:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:13.688735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519389308553892:2048] VirtualId# [1:7571519389308553890:0]
2025-11-11T17:22:13.688784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:13.689504Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519393169052182:2140]) peer tries to resume nonexistent session Self# [1:7571519389308553890:0] Peer# [2:7571519388874084867:0]
2025-11-11T17:22:13.689547Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519393169052182:2140] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:13.690612Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519393603521200:2122] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:13.691497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [1:7571519393603521201:2123] held: yes
2025-11-11T17:22:13.691848Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519393169052180:2139] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:13.697806Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519393603521201:2123] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:13.697934Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519393169052180:2139] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:13.698101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:13.698148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:13.698194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519393603521205:2048]
2025-11-11T17:22:13.698243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519393603521205:2048] [node 2] ICS09 handshake done sender: [1:7571519393603521201:2123] self: [1:7571519393603521203:0] peer: [2:7571519393169052179:0] socket: 17 qp: 3366
2025-11-11T17:22:13.698283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519393603521205:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:13.698314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:13.698313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:13.698347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:13.698395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519393169052184:2048]
2025-11-11T17:22:13.698435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519393169052184:2048] [node 1] ICS09 handshake done sender: [2:7571519393169052180:2139] self: [2:7571519393169052179:0] peer: [1:7571519393603521203:0] socket: 16 qp: 3365
2025-11-11T17:22:13.698458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519393169052184:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:13.698485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:14.133728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:14.133940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:14.134155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519393169052184:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:14.134179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519393169052184:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:14.134208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:14.134243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519393169052184:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:14.137197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:14.137446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519393603521205:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:14.137468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519393603521205:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:14.137526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:14.137558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519393603521205:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:14.143766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519393169052184:2048] [node 1] ICS08 incoming handshake Self# [1:7571519393603521203:0] Peer# [2:7571519393169052179:0] Counter# 1783 LastInputSerial# 45
2025-11-11T17:22:14.144094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519393603521205:2048] [node 2] ICS08 incoming handshake Self# [2:7571519393169052179:0] Peer# [1:7571519393603521203:0] Counter# 45 LastInputSerial# 1783
2025-11-11T17:22:14.144428Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519397898488503:2125] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:14.144673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:14.144724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519393603521205:2048] [node 2] ICS09 handshake done sender: [1:7571519397898488503:2125] self: [1:7571519393603521203:0] peer: [2:7571519393169052179:0] socket: 22 qp: 3369
2025-11-11T17:22:14.145234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519393603521205:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:14.145271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:14.146523Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:14.147373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519397464019484:2142] held: no
2025-11-11T17:22:14.149212Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519397464019484:2142] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:14.149356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:14.149396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519393169052184:2048] [node 1] ICS09 handshake done sender: [2:7571519397464019484:2142] self: [2:7571519393169052179:0] peer: [1:7571519393603521203:0] socket: 23 qp: 3371
2025-11-11T17:22:14.149861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519393169052184:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:14.149895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:14.417058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:14.413931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:14.414125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:14.414412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519393603521205:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:14.414435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519393603521205:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:14.414460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:14.414498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519393603521205:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:14.417336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519393169052184:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-11T17:22:14.417366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519393169052184:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:14.417444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:14.417486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519393169052184:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:14.419595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519393169052184:2048] [node 1] ICS08 incoming handshake Self# [1:7571519393603521203:0] Peer# [2:7571519393169052179:0] Counter# 3623 LastInputSerial# 92
2025-11-11T17:22:14.423247Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519397898488508:2127] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:14.423561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:14.423610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519393603521205:2048] [node 2] ICS09 handshake done sender: [1:7571519397898488508:2127] self: [1:7571519393603521203:0] peer: [2:7571519393169052179:0] socket: 16 qp: 3372
2025-11-11T17:22:14.423996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519393603521205:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:14.424037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:14.425960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519393603521205:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:14.426012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519393603521205:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:14.426060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:14.426099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519393603521205:2048] [node 2] ICS08 incoming handshake Self# [2:7571519393169052179:0] Peer# [1:7571519393603521203:0] Counter# 92 LastInputSerial# 3623
2025-11-11T17:22:14.426311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:14.426497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519393603521205:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:14.426524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519393603521205:2048] VirtualId# [1:7571519393603521203:0]
2025-11-11T17:22:14.426552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:14.428333Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519397464019488:2145] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:14.428692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:14.428744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519393169052184:2048] [node 1] ICS09 handshake done sender: [2:7571519397464019488:2145] self: [2:7571519393169052179:0] peer: [1:7571519393603521203:0] socket: 18 qp: 3373
2025-11-11T17:22:14.429274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519393169052184:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:14.429314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:14.429562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:14.429604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519393169052184:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:14.429625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519393169052184:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:14.429653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:14.429752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:14.429920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519393169052184:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:14.429936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519393169052184:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:14.433033Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:14.437350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519397898488514:2130]) peer tries to resume nonexistent session Self# [2:7571519393169052179:0] Peer# [1:7571519393603521203:0]
2025-11-11T17:22:14.437397Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519397898488514:2130] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:14.438116Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519397464019493:2147] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:14.438977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:14.439042Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:14.439066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519393169052184:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:14.439093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519393169052184:2048] VirtualId# [2:7571519393169052179:0]
2025-11-11T17:22:14.439209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 16426
TEvUndelivered Cookie# 16427
TEvUndelivered Cookie# 16428
TEvUndelivered Cookie# 16429
TEvUndelivered Cookie# 16430
TEvUndelivered Cookie# 16431
TEvUndelivered Cookie# 16432
TEvUndelivered Cookie# 16433
TEvUndelivered Cookie# 16434
2025-11-11T17:22:14.443788Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519397898488515:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:14.443900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:14.444942Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519397464019496:2148] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:14.448296Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519397898488515:2131] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:14.448515Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519397464019496:2148] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:14.448626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:14.448666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:14.448716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519397898488518:2048]
2025-11-11T17:22:14.448781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:14.448764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519397898488518:2048] [node 2] ICS09 handshake done sender: [1:7571519397898488515:2131] self: [1:7571519397898488516:0] peer: [2:7571519397464019495:0] socket: 17 qp: 3377
2025-11-11T17:22:14.448831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:14.448795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519397898488518:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:14.448885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519397464019499:2048]
2025-11-11T17:22:14.448829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:14.448917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519397464019499:2048] [node 1] ICS09 handshake done sender: [2:7571519397464019496:2148] self: [2:7571519397464019495:0] peer: [1:7571519397898488516:0] socket: 16 qp: 3378
2025-11-11T17:22:14.448940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519397464019499:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:14.448964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
Subscribe
2025-11-11T17:22:14.538462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:14.538652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519397898488518:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
TEvNodeConnected
2025-11-11T17:22:14.538676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519397898488518:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:14.538760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:14.538789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519397898488518:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:14.541040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:14.541401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519397464019499:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:14.541421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519397464019499:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:14.541446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:14.541482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519397464019499:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:14.543353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519397464019499:2048] [node 1] ICS08 incoming handshake Self# [1:7571519397898488516:0] Peer# [2:7571519397464019495:0] Counter# 0 LastInputSerial# 0
2025-11-11T17:22:14.551509Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519397898488520:2133] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:14.551894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519397898488518:2048] [node 2] ICS08 incoming handshake Self# [2:7571519397464019495:0] Peer# [1:7571519397898488516:0] Counter# 0 LastInputSerial# 0
2025-11-11T17:22:14.556275Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519397464019502:2150] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:14.556555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:14.556610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519397464019499:2048] [node 1] ICS09 handshake done sender: [2:7571519397464019502:2150] self: [2:7571519397464019495:0] peer: [1:7571519397898488516:0] socket: 21 qp: 3380
2025-11-11T17:22:14.557353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519397464019499:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:14.557398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:14.551935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:14.551979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519397898488518:2048] [node 2] ICS09 handshake done sender: [1:7571519397898488520:2133] self: [1:7571519397898488516:0] peer: [2:7571519397464019495:0] socket: 20 qp: 3379
2025-11-11T17:22:14.552522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519397898488518:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:14.552563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:14.877850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:14.879864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:14.880137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:14.880412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519397898488518:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:14.880435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519397898488518:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:14.880501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:14.880530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519397898488518:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:14.882536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519397464019499:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:14.882574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519397464019499:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:14.882600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:14.882670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:14.882871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519397464019499:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:14.882890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519397464019499:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:14.889270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519397464019499:2048] [node 1] ICS08 incoming handshake Self# [1:7571519397898488516:0] Peer# [2:7571519397464019495:0] Counter# 2371 LastInputSerial# 59
2025-11-11T17:22:14.889886Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519397898488525:2135] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:14.890154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:14.890204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519397898488518:2048] [node 2] ICS09 handshake done sender: [1:7571519397898488525:2135] self: [1:7571519397898488516:0] peer: [2:7571519397464019495:0] socket: 16 qp: 3383
2025-11-11T17:22:14.890699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519397898488518:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:14.890738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:14.894237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519397898488518:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:14.894264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519397898488518:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:14.894331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:14.894375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519397898488518:2048] [node 2] ICS08 incoming handshake Self# [2:7571519397464019495:0] Peer# [1:7571519397898488516:0] Counter# 59 LastInputSerial# 2371
2025-11-11T17:22:14.897704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:14.897933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519397898488518:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:14.897965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519397898488518:2048] VirtualId# [1:7571519397898488516:0]
2025-11-11T17:22:14.897999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:14.899757Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519397464019507:2153] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:14.900285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:14.900334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519397464019499:2048] [node 1] ICS09 handshake done sender: [2:7571519397464019507:2153] self: [2:7571519397464019495:0] peer: [1:7571519397898488516:0] socket: 17 qp: 3384
2025-11-11T17:22:14.900898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519397464019499:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:14.900938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:14.901197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:14.901228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519397464019499:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:14.901269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519397464019499:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:14.901293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:14.901386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:14.901556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519397464019499:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:14.901573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519397464019499:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:14.917823Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:14.935314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519397898488531:2138]) peer tries to resume nonexistent session Self# [2:7571519397464019495:0] Peer# [1:7571519397898488516:0]
2025-11-11T17:22:14.935365Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519397898488531:2138] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
TEvUndelivered Cookie# 16990
TEvUndelivered Cookie# 16991
TEvUndelivered Cookie# 16992
TEvUndelivered Cookie# 16993
TEvUndelivered Cookie# 16994
TEvUndelivered Cookie# 16995
2025-11-11T17:22:14.936888Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519397464019512:2155] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:14.937565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:14.937646Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:14.937669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519397464019499:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:14.937696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519397464019499:2048] VirtualId# [2:7571519397464019495:0]
2025-11-11T17:22:14.937802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:14.950140Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519397898488532:2139] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:14.950266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:14.961777Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519397464019515:2156] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:14.985808Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519397898488532:2139] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:14.986067Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519397464019515:2156] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:14.986131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:14.986173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:14.986215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519397898488535:2048]
2025-11-11T17:22:14.986249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519397898488535:2048] [node 2] ICS09 handshake done sender: [1:7571519397898488532:2139] self: [1:7571519397898488533:0] peer: [2:7571519397464019514:0] socket: 17 qp: 3388
2025-11-11T17:22:14.986285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519397898488535:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:14.986328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:14.986350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:14.986378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:14.986420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519397464019518:2048]
2025-11-11T17:22:14.986450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519397464019518:2048] [node 1] ICS09 handshake done sender: [2:7571519397464019515:2156] self: [2:7571519397464019514:0] peer: [1:7571519397898488533:0] socket: 16 qp: 3389
2025-11-11T17:22:14.986468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519397464019518:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:14.986488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:15.470472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519397464019518:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:15.470514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519397464019518:2048] VirtualId# [2:7571519397464019514:0]
2025-11-11T17:22:15.470542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519397464019518:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:15.470587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:15.470628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:15.471907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:15.472136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519397898488535:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:15.472156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519397898488535:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:15.472214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:15.472247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519397898488535:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:15.476168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519401758986816:2157]) peer tries to resume nonexistent session Self# [1:7571519397898488533:0] Peer# [2:7571519397464019514:0]
2025-11-11T17:22:15.476224Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519401758986816:2157] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:15.476791Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519402193455833:2141] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:15.477636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:15.477704Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:15.477727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519397898488535:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:15.477754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519397898488535:2048] VirtualId# [1:7571519397898488533:0]
2025-11-11T17:22:15.477797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:15.482381Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519401758986817:2158] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:15.482517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:15.482855Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519402193455836:2142] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:15.489815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519401758986817:2158] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:15.490036Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519402193455836:2142] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:15.490280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:15.490338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:15.490375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519402193455839:2048]
2025-11-11T17:22:15.490412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519402193455839:2048] [node 2] ICS09 handshake done sender: [1:7571519402193455836:2142] self: [1:7571519402193455835:0] peer: [2:7571519401758986818:0] socket: 16 qp: 3392
2025-11-11T17:22:15.490440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519402193455839:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:15.490471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:15.490940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:15.490973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:15.490999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519401758986820:2048]
2025-11-11T17:22:15.491026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519401758986820:2048] [node 1] ICS09 handshake done sender: [2:7571519401758986817:2158] self: [2:7571519401758986818:0] peer: [1:7571519402193455835:0] socket: 17 qp: 3391
2025-11-11T17:22:15.491047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519401758986820:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:15.491069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:15.782013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519401758986820:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-11T17:22:15.782062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519401758986820:2048] VirtualId# [2:7571519401758986818:0]
2025-11-11T17:22:15.782090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519401758986820:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:15.782156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:15.782216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:15.783530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:15.783758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519402193455839:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:15.783783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519402193455839:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:15.783853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:15.783884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519402193455839:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 18291
TEvUndelivered Cookie# 18292
2025-11-11T17:22:15.788140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519401758986822:2160]) peer tries to resume nonexistent session Self# [1:7571519402193455835:0] Peer# [2:7571519401758986818:0]
2025-11-11T17:22:15.788199Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519401758986822:2160] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:15.788879Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519402193455841:2143] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:15.790253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:15.790319Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:15.790343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519402193455839:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:15.790372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519402193455839:2048] VirtualId# [1:7571519402193455835:0]
2025-11-11T17:22:15.790416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:15.798176Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519402193455843:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:15.798738Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519401758986826:2162] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:15.798843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:15.799256Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519402193455845:2145] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:15.805887Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519401758986826:2162] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:15.806144Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519402193455845:2145] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:15.806374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:15.806431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:15.806469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519402193455849:2048]
2025-11-11T17:22:15.806506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519402193455849:2048] [node 2] ICS09 handshake done sender: [1:7571519402193455845:2145] self: [1:7571519402193455844:0] peer: [2:7571519401758986827:0] socket: 16 qp: 3397
2025-11-11T17:22:15.806534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519402193455849:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:15.806562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:15.807933Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:15.808992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:15.809041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:15.809078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519401758986829:2048]
2025-11-11T17:22:15.809112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519401758986829:2048] [node 1] ICS09 handshake done sender: [2:7571519401758986826:2162] self: [2:7571519401758986827:0] peer: [1:7571519402193455844:0] socket: 17 qp: 3396
2025-11-11T17:22:15.809140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519401758986829:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:15.809166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:15.809598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:16.129967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:16.130180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519401758986829:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-11T17:22:16.130206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519401758986829:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:16.130271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:16.130333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519401758986829:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:16.132351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:16.132633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519402193455849:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:16.132661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519402193455849:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:16.132722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:16.132753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519402193455849:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:16.139113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519401758986829:2048] [node 1] ICS08 incoming handshake Self# [1:7571519402193455844:0] Peer# [2:7571519401758986827:0] Counter# 1126 LastInputSerial# 28
2025-11-11T17:22:16.140884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519402193455849:2048] [node 2] ICS08 incoming handshake Self# [2:7571519401758986827:0] Peer# [1:7571519402193455844:0] Counter# 28 LastInputSerial# 1126
2025-11-11T17:22:16.146475Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519406488423147:2146] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:16.146739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:16.146799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519402193455849:2048] [node 2] ICS09 handshake done sender: [1:7571519406488423147:2146] self: [1:7571519402193455844:0] peer: [2:7571519401758986827:0] socket: 22 qp: 3399
2025-11-11T17:22:16.147199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519402193455849:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:16.147243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:16.150882Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:16.151882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519406053954129:2165] held: no
2025-11-11T17:22:16.151947Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519406053954129:2165] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:16.157906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:16.157970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519401758986829:2048] [node 1] ICS09 handshake done sender: [2:7571519406053954129:2165] self: [2:7571519401758986827:0] peer: [1:7571519402193455844:0] socket: 23 qp: 3401
2025-11-11T17:22:16.158657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519401758986829:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:16.158705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:16.329961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:16.330301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519401758986829:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-11T17:22:16.330328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519401758986829:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:16.330389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:16.330417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519401758986829:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:16.333572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:16.333844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519402193455849:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:16.333870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519402193455849:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:16.333937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:16.333967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519402193455849:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:16.349088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519402193455849:2048] [node 2] ICS08 incoming handshake Self# [2:7571519401758986827:0] Peer# [1:7571519402193455844:0] Counter# 46 LastInputSerial# 1884
2025-11-11T17:22:16.357890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519401758986829:2048] [node 1] ICS08 incoming handshake Self# [1:7571519402193455844:0] Peer# [2:7571519401758986827:0] Counter# 1884 LastInputSerial# 46
2025-11-11T17:22:16.358516Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519406488423152:2148] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:16.360475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:16.360561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519402193455849:2048] [node 2] ICS09 handshake done sender: [1:7571519406488423152:2148] self: [1:7571519402193455844:0] peer: [2:7571519401758986827:0] socket: 18 qp: 3403
2025-11-11T17:22:16.361065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519402193455849:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:16.361107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:16.366556Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519406053954134:2168] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:16.367179Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:16.368045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:16.368095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519401758986829:2048] [node 1] ICS09 handshake done sender: [2:7571519406053954134:2168] self: [2:7571519401758986827:0] peer: [1:7571519402193455844:0] socket: 19 qp: 3405
2025-11-11T17:22:16.368483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519401758986829:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:16.368518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:16.368634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:16.811564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:16.811717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:16.811738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519401758986829:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:16.811759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519401758986829:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:16.811781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:16.812125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:16.812362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519402193455849:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:16.812382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519402193455849:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:16.812439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:16.812468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519402193455849:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:16.814398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:16.814603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519401758986829:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:16.814621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519401758986829:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:16.816762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519401758986829:2048] [node 1] ICS08 incoming handshake Self# [1:7571519402193455844:0] Peer# [2:7571519401758986827:0] Counter# 3920 LastInputSerial# 98
2025-11-11T17:22:16.819150Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519406488423157:2150] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:16.819449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:16.819496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519402193455849:2048] [node 2] ICS09 handshake done sender: [1:7571519406488423157:2150] self: [1:7571519402193455844:0] peer: [2:7571519401758986827:0] socket: 16 qp: 3406
2025-11-11T17:22:16.819979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519402193455849:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:16.820032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:16.823014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519402193455849:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:16.823043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519402193455849:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:16.823084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:16.823121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519402193455849:2048] [node 2] ICS08 incoming handshake Self# [2:7571519401758986827:0] Peer# [1:7571519402193455844:0] Counter# 98 LastInputSerial# 3920
2025-11-11T17:22:16.823373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:16.823566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519402193455849:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:16.823593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519402193455849:2048] VirtualId# [1:7571519402193455844:0]
2025-11-11T17:22:16.823626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:16.825786Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519406053954138:2171] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:16.826138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:16.826189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519401758986829:2048] [node 1] ICS09 handshake done sender: [2:7571519406053954138:2171] self: [2:7571519401758986827:0] peer: [1:7571519402193455844:0] socket: 17 qp: 3407
2025-11-11T17:22:16.826713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519401758986829:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:16.826752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:16.827012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:16.827031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519401758986829:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:16.827050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519401758986829:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:16.827067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:16.827133Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:16.827289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519401758986829:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:16.827321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519401758986829:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:16.830851Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:16.835707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519406488423163:2153]) peer tries to resume nonexistent session Self# [2:7571519401758986827:0] Peer# [1:7571519402193455844:0]
2025-11-11T17:22:16.835761Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519406488423163:2153] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:16.836510Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519406053954143:2173] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:16.837438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:16.837521Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:16.837552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519401758986829:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:16.837581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519401758986829:2048] VirtualId# [2:7571519401758986827:0]
2025-11-11T17:22:16.841123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 19254
TEvUndelivered Cookie# 19255
TEvUndelivered Cookie# 19256
TEvUndelivered Cookie# 19257
TEvUndelivered Cookie# 19258
TEvUndelivered Cookie# 19259
TEvUndelivered Cookie# 19260
TEvUndelivered Cookie# 19261
2025-11-11T17:22:16.850438Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519406488423164:2154] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:16.850563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:16.851983Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519406053954146:2174] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:16.861928Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519406488423164:2154] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:16.863140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:16.863175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:16.863219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519406488423167:2048]
2025-11-11T17:22:16.863251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519406488423167:2048] [node 2] ICS09 handshake done sender: [1:7571519406488423164:2154] self: [1:7571519406488423165:0] peer: [2:7571519406053954145:0] socket: 17 qp: 3411
2025-11-11T17:22:16.863275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519406488423167:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:16.863313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:16.862209Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519406053954146:2174] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:16.862488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:16.862530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:16.862569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519406053954149:2048]
2025-11-11T17:22:16.862607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519406053954149:2048] [node 1] ICS09 handshake done sender: [2:7571519406053954146:2174] self: [2:7571519406053954145:0] peer: [1:7571519406488423165:0] socket: 16 qp: 3412
2025-11-11T17:22:16.862654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519406053954149:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:16.862703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:17.165272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519406488423167:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-11T17:22:17.165314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519406488423167:2048] VirtualId# [1:7571519406488423165:0]
2025-11-11T17:22:17.165342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519406488423167:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:17.165401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:17.165439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:17.165700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:17.166067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519406053954149:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:17.166089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519406053954149:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:17.166127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:17.166167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519406053954149:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:17.170442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519410783390465:2156]) peer tries to resume nonexistent session Self# [2:7571519406053954145:0] Peer# [1:7571519406488423165:0]
2025-11-11T17:22:17.170499Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519410783390465:2156] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:17.171205Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519410348921447:2175] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:17.172211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:17.172297Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:17.172322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519406053954149:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:17.172351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519406053954149:2048] VirtualId# [2:7571519406053954145:0]
2025-11-11T17:22:17.172489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 19602
TEvUndelivered Cookie# 19603
TEvUndelivered Cookie# 19604
TEvUndelivered Cookie# 19605
TEvUndelivered Cookie# 19606
TEvUndelivered Cookie# 19607
TEvUndelivered Cookie# 19608
TEvUndelivered Cookie# 19609
TEvUndelivered Cookie# 19610
TEvUndelivered Cookie# 19611
2025-11-11T17:22:17.181189Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519410348921450:2176] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:17.180774Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519410783390466:2157] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:17.180923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:17.185423Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519410783390466:2157] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:17.186311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:17.186345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:17.186374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519410783390469:2048]
2025-11-11T17:22:17.186405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519410783390469:2048] [node 2] ICS09 handshake done sender: [1:7571519410783390466:2157] self: [1:7571519410783390467:0] peer: [2:7571519410348921449:0] socket: 17 qp: 3414
2025-11-11T17:22:17.186426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519410783390469:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:17.186447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:17.185566Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519410348921450:2176] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:17.185779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:17.185816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:17.185850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519410348921453:2048]
2025-11-11T17:22:17.185885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519410348921453:2048] [node 1] ICS09 handshake done sender: [2:7571519410348921450:2176] self: [2:7571519410348921449:0] peer: [1:7571519410783390467:0] socket: 16 qp: 3415
2025-11-11T17:22:17.185913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519410348921453:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:17.185942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:17.693723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:17.694758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:17.694798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519410348921453:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:17.694821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519410348921453:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:17.694849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:17.697742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] socket: Connection reset by peer
2025-11-11T17:22:17.698020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519410783390469:2048] [node 2] ICS07 socket disconnect 17 reason# ECONNRESET
2025-11-11T17:22:17.698042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519410783390469:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-11T17:22:17.698075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:17.698148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519410783390469:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:17.705746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:17.705988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519410348921453:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:17.706034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519410348921453:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:17.708219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519410348921453:2048] [node 1] ICS08 incoming handshake Self# [1:7571519410783390467:0] Peer# [2:7571519410348921449:0] Counter# 2328 LastInputSerial# 60
2025-11-11T17:22:17.710684Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519410783390471:2159] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:17.720174Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519410348921456:2178] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:17.721220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:17.721280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519410348921453:2048] [node 1] ICS09 handshake done sender: [2:7571519410348921456:2178] self: [2:7571519410348921449:0] peer: [1:7571519410783390467:0] socket: 21 qp: 3417
2025-11-11T17:22:17.721807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519410348921453:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:17.721848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:17.720893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519410783390469:2048] [node 2] ICS08 incoming handshake Self# [2:7571519410348921449:0] Peer# [1:7571519410783390467:0] Counter# 60 LastInputSerial# 2328
2025-11-11T17:22:17.720945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:17.720987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519410783390469:2048] [node 2] ICS09 handshake done sender: [1:7571519410783390471:2159] self: [1:7571519410783390467:0] peer: [2:7571519410348921449:0] socket: 20 qp: 3416
2025-11-11T17:22:17.723989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519410783390469:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:17.724033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:18.097718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:18.097960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519410783390469:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:18.097982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519410783390469:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
TEvNodeDisconnected
2025-11-11T17:22:18.098039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:18.098070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519410783390469:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:18.097336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519410348921453:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-11-11T17:22:18.097395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519410348921453:2048] VirtualId# [2:7571519410348921449:0]
2025-11-11T17:22:18.097426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519410348921453:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:18.097491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:18.097530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:18.124369Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519415078357774:2162] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:18.124444Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519415078357772:2161] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:18.125505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:18.125572Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:18.125607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519410783390469:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:18.125635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519410783390469:2048] VirtualId# [1:7571519410783390467:0]
2025-11-11T17:22:18.125681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:18.123710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519414643888759:2181]) peer tries to resume nonexistent session Self# [1:7571519410783390467:0] Peer# [2:7571519410348921449:0]
2025-11-11T17:22:18.123763Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519414643888759:2181] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:18.137655Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519415078357776:2163] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:18.136422Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519414643888760:2182] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:18.136541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:18.149259Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519414643888760:2182] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:18.150399Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519415078357776:2163] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:18.150729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:18.150792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:18.150837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519415078357780:2048]
2025-11-11T17:22:18.150876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519415078357780:2048] [node 2] ICS09 handshake done sender: [1:7571519415078357776:2163] self: [1:7571519415078357775:0] peer: [2:7571519414643888761:0] socket: 16 qp: 3425
2025-11-11T17:22:18.150906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519415078357780:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:18.150939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:18.153479Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:18.154286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:18.154336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:18.154377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519414643888763:2048]
2025-11-11T17:22:18.154414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519414643888763:2048] [node 1] ICS09 handshake done sender: [2:7571519414643888760:2182] self: [2:7571519414643888761:0] peer: [1:7571519415078357775:0] socket: 18 qp: 3424
2025-11-11T17:22:18.154444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519414643888763:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:18.154475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:18.154836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:18.613893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519414643888763:2048] [node 1] ICS01 socket: 18 reason# Debug
2025-11-11T17:22:18.613946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519414643888763:2048] VirtualId# [2:7571519414643888761:0]
2025-11-11T17:22:18.613976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519414643888763:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:18.614040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:18.614086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:18.615148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:18.615407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519415078357780:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:18.615428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519415078357780:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:18.615500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:18.615531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519415078357780:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:18.621092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519414643888765:2184]) peer tries to resume nonexistent session Self# [1:7571519415078357775:0] Peer# [2:7571519414643888761:0]
2025-11-11T17:22:18.621167Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519414643888765:2184] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:18.621790Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519415078357782:2164] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:18.622742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:18.622802Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:18.622825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519415078357780:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:18.622852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519415078357780:2048] VirtualId# [1:7571519415078357775:0]
2025-11-11T17:22:18.622898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:18.630364Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519415078357786:2166] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:18.630700Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519414643888769:2186] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:18.630786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:18.631089Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519415078357785:2165] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:18.637824Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519414643888769:2186] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:18.638054Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519415078357785:2165] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:18.638308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:18.638364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:18.638409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519415078357790:2048]
2025-11-11T17:22:18.638450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519415078357790:2048] [node 2] ICS09 handshake done sender: [1:7571519415078357785:2165] self: [1:7571519415078357784:0] peer: [2:7571519414643888770:0] socket: 18 qp: 3430
2025-11-11T17:22:18.638480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519415078357790:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:18.638512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:18.639590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:18.639636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:18.639672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519414643888772:2048]
2025-11-11T17:22:18.639707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519414643888772:2048] [node 1] ICS09 handshake done sender: [2:7571519414643888769:2186] self: [2:7571519414643888770:0] peer: [1:7571519415078357784:0] socket: 19 qp: 3429
2025-11-11T17:22:18.639734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519414643888772:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:18.639759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:19.194575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:19.194839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519415078357790:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:19.194862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519415078357790:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:19.194907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:19.194935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519415078357790:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:19.197217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:19.197423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519414643888772:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:19.197445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519414643888772:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:19.197494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:19.197523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519414643888772:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:19.199504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519414643888772:2048] [node 1] ICS08 incoming handshake Self# [1:7571519415078357784:0] Peer# [2:7571519414643888770:0] Counter# 4030 LastInputSerial# 97
2025-11-11T17:22:19.208709Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519419373325088:2167] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:19.209075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519415078357790:2048] [node 2] ICS08 incoming handshake Self# [2:7571519414643888770:0] Peer# [1:7571519415078357784:0] Counter# 97 LastInputSerial# 4030
2025-11-11T17:22:19.209120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:19.209168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519415078357790:2048] [node 2] ICS09 handshake done sender: [1:7571519419373325088:2167] self: [1:7571519415078357784:0] peer: [2:7571519414643888770:0] socket: 16 qp: 3431
2025-11-11T17:22:19.209612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519415078357790:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:19.209654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:19.211375Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:19.212219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519418938856071:2189] held: no
2025-11-11T17:22:19.212278Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519418938856071:2189] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:19.212419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:19.212455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519414643888772:2048] [node 1] ICS09 handshake done sender: [2:7571519418938856071:2189] self: [2:7571519414643888770:0] peer: [1:7571519415078357784:0] socket: 17 qp: 3432
2025-11-11T17:22:19.212976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519414643888772:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:19.213010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:19.365990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:19.365718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:19.366273Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:19.366296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519414643888772:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:19.366339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519414643888772:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:19.366371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:19.366425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519414643888772:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:19.366440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519414643888772:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:19.369736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] socket: Connection reset by peer
2025-11-11T17:22:19.369975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519415078357790:2048] [node 2] ICS07 socket disconnect 16 reason# ECONNRESET
2025-11-11T17:22:19.369998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519415078357790:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-11T17:22:19.370025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:19.370084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519415078357790:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:19.382123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519415078357790:2048] [node 2] ICS08 incoming handshake Self# [2:7571519414643888770:0] Peer# [1:7571519415078357784:0] Counter# 120 LastInputSerial# 4937
2025-11-11T17:22:19.382359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519414643888772:2048] [node 1] ICS08 incoming handshake Self# [1:7571519415078357784:0] Peer# [2:7571519414643888770:0] Counter# 4937 LastInputSerial# 120
2025-11-11T17:22:19.384143Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519419373325093:2169] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:19.384392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:19.384447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519415078357790:2048] [node 2] ICS09 handshake done sender: [1:7571519419373325093:2169] self: [1:7571519415078357784:0] peer: [2:7571519414643888770:0] socket: 20 qp: 3436
2025-11-11T17:22:19.384949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519415078357790:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:19.384988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:19.386963Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519418938856077:2192] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:19.387152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:19.387198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519414643888772:2048] [node 1] ICS09 handshake done sender: [2:7571519418938856077:2192] self: [2:7571519414643888770:0] peer: [1:7571519415078357784:0] socket: 21 qp: 3438
2025-11-11T17:22:19.387742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519414643888772:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:19.387777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:19.906057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:19.906333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519414643888772:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-11-11T17:22:19.906359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519414643888772:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:19.906399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:19.906428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519414643888772:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:19.908571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:19.908801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519415078357790:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:19.908819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519415078357790:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:19.908878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:19.908906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519415078357790:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:19.916072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519414643888772:2048] [node 1] ICS08 incoming handshake Self# [1:7571519415078357784:0] Peer# [2:7571519414643888770:0] Counter# 8753 LastInputSerial# 214
2025-11-11T17:22:19.916514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519415078357790:2048] [node 2] ICS08 incoming handshake Self# [2:7571519414643888770:0] Peer# [1:7571519415078357784:0] Counter# 214 LastInputSerial# 8753
2025-11-11T17:22:19.917067Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519419373325098:2171] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:19.917329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:19.917378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519415078357790:2048] [node 2] ICS09 handshake done sender: [1:7571519419373325098:2171] self: [1:7571519415078357784:0] peer: [2:7571519414643888770:0] socket: 18 qp: 3440
2025-11-11T17:22:19.917797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519415078357790:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:19.917833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:19.922507Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:19.923477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519418938856082:2195] held: no
2025-11-11T17:22:19.923540Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519418938856082:2195] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:19.923960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:19.924004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519414643888772:2048] [node 1] ICS09 handshake done sender: [2:7571519418938856082:2195] self: [2:7571519414643888770:0] peer: [1:7571519415078357784:0] socket: 19 qp: 3442
2025-11-11T17:22:19.924467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519414643888772:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:19.924509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:20.316715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:20.316945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519415078357790:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:20.316967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519415078357790:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:20.317008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:20.317037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519415078357790:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:20.319196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:20.319561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519414643888772:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:20.319584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519414643888772:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:20.319605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:20.319640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519414643888772:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:20.321837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519414643888772:2048] [node 1] ICS08 incoming handshake Self# [1:7571519415078357784:0] Peer# [2:7571519414643888770:0] Counter# 11638 LastInputSerial# 285
2025-11-11T17:22:20.328096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519415078357790:2048] [node 2] ICS08 incoming handshake Self# [2:7571519414643888770:0] Peer# [1:7571519415078357784:0] Counter# 285 LastInputSerial# 11638
2025-11-11T17:22:20.329793Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519423668292399:2173] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:20.329970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:20.330013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519415078357790:2048] [node 2] ICS09 handshake done sender: [1:7571519423668292399:2173] self: [1:7571519415078357784:0] peer: [2:7571519414643888770:0] socket: 16 qp: 3443
2025-11-11T17:22:20.330393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519415078357790:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:20.330417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:20.331918Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:20.332663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519423233823382:2198] held: no
2025-11-11T17:22:20.332730Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519423233823382:2198] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:20.332848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:20.332893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519414643888772:2048] [node 1] ICS09 handshake done sender: [2:7571519423233823382:2198] self: [2:7571519414643888770:0] peer: [1:7571519415078357784:0] socket: 17 qp: 3444
2025-11-11T17:22:20.333398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519414643888772:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:20.333438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:20.488944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:20.490580Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:20.490765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519414643888772:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:20.490785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519414643888772:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:20.490807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:20.490857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519414643888772:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:20.489469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:20.489875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519415078357790:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:20.489901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519415078357790:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:20.489959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:20.489988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519415078357790:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:20.498930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519414643888772:2048] [node 1] ICS08 incoming handshake Self# [1:7571519415078357784:0] Peer# [2:7571519414643888770:0] Counter# 12562 LastInputSerial# 308
2025-11-11T17:22:20.504749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519415078357790:2048] [node 2] ICS08 incoming handshake Self# [2:7571519414643888770:0] Peer# [1:7571519415078357784:0] Counter# 308 LastInputSerial# 12562
2025-11-11T17:22:20.511420Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519423668292404:2175] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:20.511874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:20.511931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519415078357790:2048] [node 2] ICS09 handshake done sender: [1:7571519423668292404:2175] self: [1:7571519415078357784:0] peer: [2:7571519414643888770:0] socket: 18 qp: 3449
2025-11-11T17:22:20.512466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519415078357790:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:20.512512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:20.514666Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519423233823388:2201] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:20.514972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:20.515050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519414643888772:2048] [node 1] ICS09 handshake done sender: [2:7571519423233823388:2201] self: [2:7571519414643888770:0] peer: [1:7571519415078357784:0] socket: 20 qp: 3448
2025-11-11T17:22:20.515724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519414643888772:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:20.515783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:20.713780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519414643888772:2048] [node 1] ICS01 socket: 20 reason# Debug
2025-11-11T17:22:20.713830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519414643888772:2048] VirtualId# [2:7571519414643888770:0]
2025-11-11T17:22:20.713860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519414643888772:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:20.713911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:20.713965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:20.714823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:20.715083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519415078357790:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:20.715106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519415078357790:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:20.715165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:20.715196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519415078357790:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 25055
2025-11-11T17:22:20.730820Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519423668292409:2177] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:20.726248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519423233823391:2203]) peer tries to resume nonexistent session Self# [1:7571519415078357784:0] Peer# [2:7571519414643888770:0]
2025-11-11T17:22:20.726317Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519423233823391:2203] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:20.732209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:20.732276Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:20.732300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519415078357790:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:20.732328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519415078357790:2048] VirtualId# [1:7571519415078357784:0]
2025-11-11T17:22:20.732374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:20.743491Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519423668292411:2178] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:20.749824Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519423233823395:2205] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:20.749946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:20.752042Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519423668292413:2179] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:20.765348Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519423233823395:2205] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:20.771104Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519423668292413:2179] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:20.772569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:20.772627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:20.772681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519423233823398:2048]
2025-11-11T17:22:20.772720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519423233823398:2048] [node 1] ICS09 handshake done sender: [2:7571519423233823395:2205] self: [2:7571519423233823396:0] peer: [1:7571519423668292412:0] socket: 17 qp: 3454
2025-11-11T17:22:20.772760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519423233823398:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:20.772791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:20.778261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:20.778325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:20.778370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519423668292417:2048]
2025-11-11T17:22:20.778407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519423668292417:2048] [node 2] ICS09 handshake done sender: [1:7571519423668292413:2179] self: [1:7571519423668292412:0] peer: [2:7571519423233823396:0] socket: 16 qp: 3455
2025-11-11T17:22:20.778437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519423668292417:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:20.778467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:20.875322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:20.875628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519423233823398:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-11T17:22:20.875660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519423233823398:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:20.875702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:20.875732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519423233823398:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:20.876463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:20.876657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519423668292417:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:20.876678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519423668292417:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:20.876724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:20.876754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519423668292417:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:20.882106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519423668292417:2048] [node 2] ICS08 incoming handshake Self# [2:7571519423233823396:0] Peer# [1:7571519423668292412:0] Counter# 7 LastInputSerial# 274
2025-11-11T17:22:20.887051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519423233823398:2048] [node 1] ICS08 incoming handshake Self# [1:7571519423668292412:0] Peer# [2:7571519423233823396:0] Counter# 274 LastInputSerial# 7
2025-11-11T17:22:20.889037Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519423668292419:2180] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:20.889390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:20.889446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519423668292417:2048] [node 2] ICS09 handshake done sender: [1:7571519423668292419:2180] self: [1:7571519423668292412:0] peer: [2:7571519423233823396:0] socket: 22 qp: 3459
2025-11-11T17:22:20.889996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519423668292417:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:20.890036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:20.897152Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:20.898210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519423233823402:2208] held: no
2025-11-11T17:22:20.898276Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519423233823402:2208] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:20.898420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:20.898469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519423233823398:2048] [node 1] ICS09 handshake done sender: [2:7571519423233823402:2208] self: [2:7571519423233823396:0] peer: [1:7571519423668292412:0] socket: 23 qp: 3458
2025-11-11T17:22:20.898861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519423233823398:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:20.898903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:21.109058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:21.109340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519423668292417:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-11-11T17:22:21.109362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519423668292417:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:21.109394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:21.109421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519423668292417:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:21.110826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:21.111019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519423233823398:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-11T17:22:21.111040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519423233823398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:21.111087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:21.111120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519423233823398:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:21.116131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519423233823398:2048] [node 1] ICS08 incoming handshake Self# [1:7571519423668292412:0] Peer# [2:7571519423233823396:0] Counter# 2006 LastInputSerial# 49
2025-11-11T17:22:21.117567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519423668292417:2048] [node 2] ICS08 incoming handshake Self# [2:7571519423233823396:0] Peer# [1:7571519423668292412:0] Counter# 49 LastInputSerial# 2006
2025-11-11T17:22:21.117928Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519427963259720:2182] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:21.119749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:21.119807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519423668292417:2048] [node 2] ICS09 handshake done sender: [1:7571519427963259720:2182] self: [1:7571519423668292412:0] peer: [2:7571519423233823396:0] socket: 16 qp: 3461
2025-11-11T17:22:21.120322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519423668292417:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:21.120364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:21.122071Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519427528790702:2211] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:21.122654Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:21.123482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:21.123534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519423233823398:2048] [node 1] ICS09 handshake done sender: [2:7571519427528790702:2211] self: [2:7571519423233823396:0] peer: [1:7571519423668292412:0] socket: 17 qp: 3460
2025-11-11T17:22:21.124139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519423233823398:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:21.124182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:21.124292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:21.517779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:21.517979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519423233823398:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-11T17:22:21.518004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519423233823398:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:21.518040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:21.518069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519423233823398:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:21.519164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:21.519413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519423668292417:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:21.519435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519423668292417:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:21.519489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:21.519520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519423668292417:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:21.523636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519423668292417:2048] [node 2] ICS08 incoming handshake Self# [2:7571519423233823396:0] Peer# [1:7571519423668292412:0] Counter# 132 LastInputSerial# 5295
2025-11-11T17:22:21.527207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519423233823398:2048] [node 1] ICS08 incoming handshake Self# [1:7571519423668292412:0] Peer# [2:7571519423233823396:0] Counter# 5295 LastInputSerial# 132
2025-11-11T17:22:21.529347Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519427963259725:2184] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:21.529667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:21.529727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519423668292417:2048] [node 2] ICS09 handshake done sender: [1:7571519427963259725:2184] self: [1:7571519423668292412:0] peer: [2:7571519423233823396:0] socket: 22 qp: 3467
2025-11-11T17:22:21.531037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519423668292417:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:21.531083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:21.533231Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519427528790708:2214] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:21.533464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:21.533523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519423233823398:2048] [node 1] ICS09 handshake done sender: [2:7571519427528790708:2214] self: [2:7571519423233823396:0] peer: [1:7571519423668292412:0] socket: 23 qp: 3466
2025-11-11T17:22:21.534181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519423233823398:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:21.534224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:21.789907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:21.790242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519423233823398:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-11T17:22:21.790265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519423233823398:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:21.790301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:21.790330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519423233823398:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:21.792557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:21.792791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519423668292417:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:21.792813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519423668292417:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:21.792866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:21.792897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519423668292417:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:21.800558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519423233823398:2048] [node 1] ICS08 incoming handshake Self# [1:7571519423668292412:0] Peer# [2:7571519423233823396:0] Counter# 7291 LastInputSerial# 184
2025-11-11T17:22:21.805907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519423668292417:2048] [node 2] ICS08 incoming handshake Self# [2:7571519423233823396:0] Peer# [1:7571519423668292412:0] Counter# 184 LastInputSerial# 7291
2025-11-11T17:22:21.806382Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519427963259730:2186] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:21.806647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:21.806704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519423668292417:2048] [node 2] ICS09 handshake done sender: [1:7571519427963259730:2186] self: [1:7571519423668292412:0] peer: [2:7571519423233823396:0] socket: 18 qp: 3469
2025-11-11T17:22:21.807206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519423668292417:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:21.807245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:21.808785Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:21.810036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519427528790713:2217] held: no
2025-11-11T17:22:21.819167Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519427528790713:2217] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:21.819565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:21.819624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519423233823398:2048] [node 1] ICS09 handshake done sender: [2:7571519427528790713:2217] self: [2:7571519423233823396:0] peer: [1:7571519423668292412:0] socket: 19 qp: 3471
2025-11-11T17:22:21.820177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519423233823398:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:21.820242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:22.234539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:22.234804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519423668292417:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:22.234830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519423668292417:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:22.234878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:22.234907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519423668292417:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:22.235583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:22.235818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519423233823398:2048] [node 1] ICS01 socket: 19 reason# EndOfStream
2025-11-11T17:22:22.235850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519423233823398:2048] VirtualId# [2:7571519423233823396:0]
2025-11-11T17:22:22.235900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519423233823398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:22.235959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:22.236007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:22.242185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519431823758012:2219]) peer tries to resume nonexistent session Self# [1:7571519423668292412:0] Peer# [2:7571519423233823396:0]
2025-11-11T17:22:22.242281Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519431823758012:2219] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:22.243160Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519432258227031:2188] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:22.244260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:22.244333Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:22.244361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519423668292417:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:22.244395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519423668292417:2048] VirtualId# [1:7571519423668292412:0]
2025-11-11T17:22:22.244449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:22.253847Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519431823758016:2221] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:22.254059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:22.254469Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519432258227036:2190] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:22.254562Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519432258227034:2189] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:22.261815Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519431823758016:2221] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:22.269740Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519432258227034:2189] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:22.269929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:22.269992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:22.270034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519431823758019:2048]
2025-11-11T17:22:22.270073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519431823758019:2048] [node 1] ICS09 handshake done sender: [2:7571519431823758016:2221] self: [2:7571519431823758017:0] peer: [1:7571519432258227033:0] socket: 17 qp: 3473
2025-11-11T17:22:22.270104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519431823758019:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:22.270138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:22.272709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:22.272770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:22.272811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519432258227039:2048]
2025-11-11T17:22:22.272850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519432258227039:2048] [node 2] ICS09 handshake done sender: [1:7571519432258227034:2189] self: [1:7571519432258227033:0] peer: [2:7571519431823758017:0] socket: 16 qp: 3474
2025-11-11T17:22:22.272879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519432258227039:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:22.272916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:22.818357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519432258227039:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:22.818403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519432258227039:2048] VirtualId# [1:7571519432258227033:0]
2025-11-11T17:22:22.818433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519432258227039:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:22.818487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:22.818519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:22.818728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:22.818949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519431823758019:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:22.818971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519431823758019:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:22.819058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:22.819097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519431823758019:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:22.823328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519432258227041:2191]) peer tries to resume nonexistent session Self# [2:7571519431823758017:0] Peer# [1:7571519432258227033:0]
2025-11-11T17:22:22.823949Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519431823758021:2223] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:22.823395Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519432258227041:2191] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:22.826170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:22.826240Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:22.826263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519431823758019:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:22.826291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519431823758019:2048] VirtualId# [2:7571519431823758017:0]
2025-11-11T17:22:22.826345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:22.830082Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519431823758023:2224] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:22.831720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:22.832149Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519432258227043:2192] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:22.832317Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519432258227045:2193] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:22.835322Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519431823758023:2224] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:22.836131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:22.836179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:22.836216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519431823758029:2048]
2025-11-11T17:22:22.836248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519431823758029:2048] [node 1] ICS09 handshake done sender: [2:7571519431823758023:2224] self: [2:7571519431823758027:0] peer: [1:7571519432258227042:0] socket: 20 qp: 3479
2025-11-11T17:22:22.836314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519431823758029:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:22.836344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:22.835489Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519432258227043:2192] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:22.835685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:22.835732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:22.835771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519432258227048:2048]
2025-11-11T17:22:22.835807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519432258227048:2048] [node 2] ICS09 handshake done sender: [1:7571519432258227043:2192] self: [1:7571519432258227042:0] peer: [2:7571519431823758027:0] socket: 19 qp: 3478
2025-11-11T17:22:22.835837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519432258227048:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:22.835864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:23.317801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:23.319536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:23.319562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519431823758029:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:23.319590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519431823758029:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:23.319618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:23.319914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:23.320210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519432258227048:2048] [node 2] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:23.320244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519432258227048:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:23.320309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:23.320346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519432258227048:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:23.321805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:23.322011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519431823758029:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:23.322032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519431823758029:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:23.324736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519431823758029:2048] [node 1] ICS08 incoming handshake Self# [1:7571519432258227042:0] Peer# [2:7571519431823758027:0] Counter# 2853 LastInputSerial# 73
2025-11-11T17:22:23.326681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519436553194346:2194] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:23.326998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:23.327039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519432258227048:2048] [node 2] ICS09 handshake done sender: [1:7571519436553194346:2194] self: [1:7571519432258227042:0] peer: [2:7571519431823758027:0] socket: 16 qp: 3482
2025-11-11T17:22:23.327697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519432258227048:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:23.327736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:23.327835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519432258227048:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:23.327858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519432258227048:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:23.327902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:23.327939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519432258227048:2048] [node 2] ICS08 incoming handshake Self# [2:7571519431823758027:0] Peer# [1:7571519432258227042:0] Counter# 73 LastInputSerial# 2853
2025-11-11T17:22:23.328005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:23.328184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519432258227048:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:23.328211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519432258227048:2048] VirtualId# [1:7571519432258227042:0]
2025-11-11T17:22:23.328236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:23.331506Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519436118725328:2228] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:23.331858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:23.331908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519431823758029:2048] [node 1] ICS09 handshake done sender: [2:7571519436118725328:2228] self: [2:7571519431823758027:0] peer: [1:7571519432258227042:0] socket: 17 qp: 3483
2025-11-11T17:22:23.332523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519431823758029:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:23.332562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:23.332813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:23.332834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519431823758029:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:23.332856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519431823758029:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:23.332875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:23.332940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:23.333101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519431823758029:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:23.333116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519431823758029:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:23.336213Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:23.341104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519436553194352:2197]) peer tries to resume nonexistent session Self# [2:7571519431823758027:0] Peer# [1:7571519432258227042:0]
2025-11-11T17:22:23.341168Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519436553194352:2197] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:23.341968Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519436118725333:2230] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:23.342975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:23.343060Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:23.343085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519431823758029:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:23.343114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519431823758029:2048] VirtualId# [2:7571519431823758027:0]
2025-11-11T17:22:23.343235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 29446
TEvUndelivered Cookie# 29447
TEvUndelivered Cookie# 29448
TEvUndelivered Cookie# 29449
TEvUndelivered Cookie# 29450
TEvUndelivered Cookie# 29451
TEvUndelivered Cookie# 29452
2025-11-11T17:22:23.348877Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519436553194353:2198] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:23.349003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:23.350203Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519436118725336:2231] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:23.357133Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519436553194353:2198] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:23.357374Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519436118725336:2231] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:23.357675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:23.357727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:23.357767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519436118725339:2048]
2025-11-11T17:22:23.357803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519436118725339:2048] [node 1] ICS09 handshake done sender: [2:7571519436118725336:2231] self: [2:7571519436118725335:0] peer: [1:7571519436553194354:0] socket: 16 qp: 3488
2025-11-11T17:22:23.357831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519436118725339:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:23.357860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:23.358354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:23.358390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:23.358444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519436553194356:2048]
2025-11-11T17:22:23.358475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519436553194356:2048] [node 2] ICS09 handshake done sender: [1:7571519436553194353:2198] self: [1:7571519436553194354:0] peer: [2:7571519436118725335:0] socket: 17 qp: 3487
2025-11-11T17:22:23.358499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519436553194356:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:23.358524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:23.597870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519436118725339:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:23.597926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519436118725339:2048] VirtualId# [2:7571519436118725335:0]
2025-11-11T17:22:23.597953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519436118725339:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:23.598011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:23.598059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:23.598995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:23.599245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519436553194356:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:23.599269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519436553194356:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:23.599353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:23.599385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519436553194356:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:23.610025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519436118725341:2232]) peer tries to resume nonexistent session Self# [1:7571519436553194354:0] Peer# [2:7571519436118725335:0]
2025-11-11T17:22:23.610112Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519436118725341:2232] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:23.610726Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519436553194358:2200] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:23.611593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:23.611661Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:23.611687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519436553194356:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:23.611715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519436553194356:2048] VirtualId# [1:7571519436553194354:0]
2025-11-11T17:22:23.611759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:23.619641Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519436118725345:2234] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:23.619846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:23.620208Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519436553194363:2202] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:23.620305Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519436553194361:2201] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:23.633938Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519436118725345:2234] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:23.634229Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519436553194361:2201] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:23.634509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:23.634572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:23.634614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519436553194366:2048]
2025-11-11T17:22:23.634650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519436553194366:2048] [node 2] ICS09 handshake done sender: [1:7571519436553194361:2201] self: [1:7571519436553194360:0] peer: [2:7571519436118725346:0] socket: 17 qp: 3491
2025-11-11T17:22:23.634681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519436553194366:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:23.634711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:23.636030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:23.636072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:23.636107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519436118725348:2048]
2025-11-11T17:22:23.636138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519436118725348:2048] [node 1] ICS09 handshake done sender: [2:7571519436118725345:2234] self: [2:7571519436118725346:0] peer: [1:7571519436553194360:0] socket: 18 qp: 3490
2025-11-11T17:22:23.636161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519436118725348:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:23.636208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:24.134446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:24.134724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519436553194366:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-11T17:22:24.134753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519436553194366:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:24.134802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:24.134834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519436553194366:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:24.137016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:24.137248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519436118725348:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:24.137271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519436118725348:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:24.137329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:24.137386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519436118725348:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:24.140870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519436118725348:2048] [node 1] ICS08 incoming handshake Self# [1:7571519436553194360:0] Peer# [2:7571519436118725346:0] Counter# 2520 LastInputSerial# 63
2025-11-11T17:22:24.141381Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519440848161664:2203] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:24.141612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:24.141659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519436553194366:2048] [node 2] ICS09 handshake done sender: [1:7571519440848161664:2203] self: [1:7571519436553194360:0] peer: [2:7571519436118725346:0] socket: 16 qp: 3494
2025-11-11T17:22:24.142262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519436553194366:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:24.142307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:24.146047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519436553194366:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:24.146079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519436553194366:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:24.146133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:24.146174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519436553194366:2048] [node 2] ICS08 incoming handshake Self# [2:7571519436118725346:0] Peer# [1:7571519436553194360:0] Counter# 63 LastInputSerial# 2520
2025-11-11T17:22:24.146470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:24.146694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519436553194366:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:24.146724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519436553194366:2048] VirtualId# [1:7571519436553194360:0]
2025-11-11T17:22:24.146757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:24.148528Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519440413692647:2237] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:24.148842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:24.148909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519436118725348:2048] [node 1] ICS09 handshake done sender: [2:7571519440413692647:2237] self: [2:7571519436118725346:0] peer: [1:7571519436553194360:0] socket: 19 qp: 3496
2025-11-11T17:22:24.149556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519436118725348:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:24.149617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:24.149917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:24.149947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519436118725348:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:24.149969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519436118725348:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:24.149990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:24.150061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:24.150239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519436118725348:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:24.150255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519436118725348:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:24.153234Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:24.157686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519440848161670:2206]) peer tries to resume nonexistent session Self# [2:7571519436118725346:0] Peer# [1:7571519436553194360:0]
2025-11-11T17:22:24.157736Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519440848161670:2206] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
2025-11-11T17:22:24.158560Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519440413692652:2239] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:24.159559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:24.159629Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:24.159653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519436118725348:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:24.159682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519436118725348:2048] VirtualId# [2:7571519436118725346:0]
2025-11-11T17:22:24.159735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:24.164684Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519440848161671:2207] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:24.164806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:24.168752Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519440413692655:2240] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:24.182009Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519440413692655:2240] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:24.182372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:24.182419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:24.182466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519440413692658:2048]
2025-11-11T17:22:24.182501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519440413692658:2048] [node 1] ICS09 handshake done sender: [2:7571519440413692655:2240] self: [2:7571519440413692654:0] peer: [1:7571519440848161672:0] socket: 16 qp: 3500
2025-11-11T17:22:24.182531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519440413692658:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:24.182560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:24.181845Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519440848161671:2207] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:24.185937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:24.185986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:24.186033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519440848161674:2048]
2025-11-11T17:22:24.186076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519440848161674:2048] [node 2] ICS09 handshake done sender: [1:7571519440848161671:2207] self: [1:7571519440848161672:0] peer: [2:7571519440413692654:0] socket: 17 qp: 3499
2025-11-11T17:22:24.186106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519440848161674:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:24.186138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:24.730626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:24.730959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519440413692658:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:24.730986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519440413692658:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:24.731035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:24.731082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519440413692658:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:24.731977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:24.732233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519440848161674:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:24.732255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519440848161674:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:24.732301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:24.732330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519440848161674:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:24.739825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519440848161674:2048] [node 2] ICS08 incoming handshake Self# [2:7571519440413692654:0] Peer# [1:7571519440848161672:0] Counter# 63 LastInputSerial# 2485
2025-11-11T17:22:24.742074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519440413692658:2048] [node 1] ICS08 incoming handshake Self# [1:7571519440848161672:0] Peer# [2:7571519440413692654:0] Counter# 2485 LastInputSerial# 63
2025-11-11T17:22:24.742672Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519440848161676:2209] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:24.742982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:24.743047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519440848161674:2048] [node 2] ICS09 handshake done sender: [1:7571519440848161676:2209] self: [1:7571519440848161672:0] peer: [2:7571519440413692654:0] socket: 22 qp: 3504
2025-11-11T17:22:24.743540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519440848161674:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:24.743583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:24.748391Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519440413692662:2242] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:24.749091Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:24.750152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:24.750269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519440413692658:2048] [node 1] ICS09 handshake done sender: [2:7571519440413692662:2242] self: [2:7571519440413692654:0] peer: [1:7571519440848161672:0] socket: 23 qp: 3503
2025-11-11T17:22:24.750908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519440413692658:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:24.750950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:24.751110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:25.088182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:25.088415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519440413692658:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-11T17:22:25.088439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519440413692658:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:25.088486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:25.088518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519440413692658:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:25.091266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:25.091549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519440848161674:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:25.091573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519440848161674:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:25.091634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:25.091665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519440848161674:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:25.103997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519440413692658:2048] [node 1] ICS08 incoming handshake Self# [1:7571519440848161672:0] Peer# [2:7571519440413692654:0] Counter# 4968 LastInputSerial# 125
2025-11-11T17:22:25.104902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519440848161674:2048] [node 2] ICS08 incoming handshake Self# [2:7571519440413692654:0] Peer# [1:7571519440848161672:0] Counter# 125 LastInputSerial# 4968
2025-11-11T17:22:25.132625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [2:7571519350219379036:2048] [node 1] ICP35 event processing took too much time StateWork:293(28524157ns) {IncomingHandshake:229(28521956ns) {DropIncomingHandshake:441(914ns) IssueIncomingHandshakeReply:193(23722ns)}}
2025-11-11T17:22:25.134385Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519445143128977:2211] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:25.135096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:25.135158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519440848161674:2048] [node 2] ICS09 handshake done sender: [1:7571519445143128977:2211] self: [1:7571519440848161672:0] peer: [2:7571519440413692654:0] socket: 18 qp: 3506
2025-11-11T17:22:25.135715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519440848161674:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:25.135761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:25.140258Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:25.141545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519444708659963:2245] held: no
2025-11-11T17:22:25.141633Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519444708659963:2245] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:25.141856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:25.141901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519440413692658:2048] [node 1] ICS09 handshake done sender: [2:7571519444708659963:2245] self: [2:7571519440413692654:0] peer: [1:7571519440848161672:0] socket: 19 qp: 3508
2025-11-11T17:22:25.142531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519440413692658:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:25.142571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:25.546065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:25.546342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519440413692658:2048] [node 1] ICS07 socket disconnect 19 reason# Debug
2025-11-11T17:22:25.546376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519440413692658:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:25.546450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:25.546486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519440413692658:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:25.549734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:25.550071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519440848161674:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:25.550104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519440848161674:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:25.550163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:25.550200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519440848161674:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:25.557796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519440848161674:2048] [node 2] ICS08 incoming handshake Self# [2:7571519440413692654:0] Peer# [1:7571519440848161672:0] Counter# 173 LastInputSerial# 6922
2025-11-11T17:22:25.561964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519440413692658:2048] [node 1] ICS08 incoming handshake Self# [1:7571519440848161672:0] Peer# [2:7571519440413692654:0] Counter# 6922 LastInputSerial# 173
2025-11-11T17:22:25.562783Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519445143128982:2213] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:25.563116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:25.563194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519440848161674:2048] [node 2] ICS09 handshake done sender: [1:7571519445143128982:2213] self: [1:7571519440848161672:0] peer: [2:7571519440413692654:0] socket: 22 qp: 3510
2025-11-11T17:22:25.564155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519440848161674:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:25.564211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:25.573427Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:25.574646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519444708659968:2248] held: no
2025-11-11T17:22:25.574729Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519444708659968:2248] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:25.574986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:25.575050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519440413692658:2048] [node 1] ICS09 handshake done sender: [2:7571519444708659968:2248] self: [2:7571519440413692654:0] peer: [1:7571519440848161672:0] socket: 23 qp: 3512
2025-11-11T17:22:25.575897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519440413692658:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:25.575949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:25.897746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:25.898300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:25.898660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [1 <-> 2] socket: Broken pipe
2025-11-11T17:22:25.898702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519440848161674:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:25.898731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519440848161674:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:25.898764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:25.898826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519440848161674:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:25.898857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519440848161674:2048] VirtualId# [1:7571519440848161672:0]
2025-11-11T17:22:25.898885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:25.906485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:25.906786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:25.906810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519440413692658:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:25.906834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519440413692658:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:25.906888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:25.906963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519440413692658:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:25.906979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519440413692658:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:25.926050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519445143128990:2216]) peer tries to resume nonexistent session Self# [2:7571519440413692654:0] Peer# [1:7571519440848161672:0]
2025-11-11T17:22:25.926106Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519445143128990:2216] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:25.929887Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519444708659971:2250] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:25.929992Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519444708659972:2251] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 32420
2025-11-11T17:22:25.936849Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519445143128988:2215] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:25.934008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:25.934099Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:25.934124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519440413692658:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:25.934154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519440413692658:2048] VirtualId# [2:7571519440413692654:0]
2025-11-11T17:22:25.934226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:25.935464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:25.954970Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519445143128991:2217] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:25.964916Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519444708659971:2250] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:25.965123Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519445143128988:2215] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:25.965439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:25.965499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:25.965542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519445143128994:2048]
2025-11-11T17:22:25.965579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519445143128994:2048] [node 2] ICS09 handshake done sender: [1:7571519445143128988:2215] self: [1:7571519445143128987:0] peer: [2:7571519444708659976:0] socket: 16 qp: 3514
2025-11-11T17:22:25.965626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519445143128994:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:25.965661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:25.967151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:25.967216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:25.967256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519444708659979:2048]
2025-11-11T17:22:25.967309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519444708659979:2048] [node 1] ICS09 handshake done sender: [2:7571519444708659971:2250] self: [2:7571519444708659976:0] peer: [1:7571519445143128987:0] socket: 17 qp: 3513
2025-11-11T17:22:25.967337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519444708659979:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:25.967367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:26.234797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:26.235842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:26.236166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [1 <-> 2] socket: Broken pipe
2025-11-11T17:22:26.236190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519445143128994:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:26.236214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519445143128994:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:26.236264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:26.236323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519445143128994:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:26.236354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519445143128994:2048] VirtualId# [1:7571519445143128987:0]
2025-11-11T17:22:26.236380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:26.239247Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:26.239476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519444708659979:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:26.239505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519444708659979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:26.243412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:26.243473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519444708659979:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:26.257975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519449438096295:2219]) peer tries to resume nonexistent session Self# [2:7571519444708659976:0] Peer# [1:7571519445143128987:0]
2025-11-11T17:22:26.258025Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519449438096295:2219] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:26.258964Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519449003627278:2255] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:26.259049Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519449003627277:2254] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:26.260233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:26.260297Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:26.260322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519444708659979:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:26.260355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519444708659979:2048] VirtualId# [2:7571519444708659976:0]
2025-11-11T17:22:26.260406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:26.261860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:26.262430Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519449438096293:2218] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:26.270814Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519449438096296:2220] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:26.286208Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519449003627278:2255] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:26.290126Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519449438096293:2218] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:26.290438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:26.290499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:26.290544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519449438096299:2048]
2025-11-11T17:22:26.290580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519449438096299:2048] [node 2] ICS09 handshake done sender: [1:7571519449438096293:2218] self: [1:7571519449438096292:0] peer: [2:7571519449003627282:0] socket: 16 qp: 3519
2025-11-11T17:22:26.290611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519449438096299:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:26.290643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:26.292112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:26.292161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:26.292191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519449003627285:2048]
2025-11-11T17:22:26.292220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519449003627285:2048] [node 1] ICS09 handshake done sender: [2:7571519449003627278:2255] self: [2:7571519449003627282:0] peer: [1:7571519449438096292:0] socket: 18 qp: 3518
2025-11-11T17:22:26.292243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519449003627285:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:26.292265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:26.697741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519449438096299:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:26.697790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519449438096299:2048] VirtualId# [1:7571519449438096292:0]
2025-11-11T17:22:26.697824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519449438096299:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:26.697900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:26.697951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:26.699461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:26.699694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519449003627285:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:26.699718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519449003627285:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:26.699778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:26.699813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519449003627285:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:26.707637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519449438096301:2221]) peer tries to resume nonexistent session Self# [2:7571519449003627282:0] Peer# [1:7571519449438096292:0]
2025-11-11T17:22:26.707708Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519449438096301:2221] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:26.708412Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519449003627287:2258] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:26.709387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:26.709453Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:26.709478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519449003627285:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:26.709521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519449003627285:2048] VirtualId# [2:7571519449003627282:0]
2025-11-11T17:22:26.709696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 33182
TEvUndelivered Cookie# 33183
TEvUndelivered Cookie# 33184
TEvUndelivered Cookie# 33185
TEvUndelivered Cookie# 33186
TEvUndelivered Cookie# 33187
TEvUndelivered Cookie# 33188
TEvUndelivered Cookie# 33189
TEvUndelivered Cookie# 33190
TEvUndelivered Cookie# 33191
2025-11-11T17:22:26.719759Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519449438096302:2222] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:26.719907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:26.720991Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519449003627290:2259] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:26.737868Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519449438096302:2222] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:26.739187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:26.739232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:26.739306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519449438096305:2048]
2025-11-11T17:22:26.739347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519449438096305:2048] [node 2] ICS09 handshake done sender: [1:7571519449438096302:2222] self: [1:7571519449438096303:0] peer: [2:7571519449003627289:0] socket: 17 qp: 3524
2025-11-11T17:22:26.739380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519449438096305:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:26.739411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:26.741242Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519449003627290:2259] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:26.745932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:26.745984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:26.746027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519449003627293:2048]
2025-11-11T17:22:26.746063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519449003627293:2048] [node 1] ICS09 handshake done sender: [2:7571519449003627290:2259] self: [2:7571519449003627289:0] peer: [1:7571519449438096303:0] socket: 16 qp: 3525
2025-11-11T17:22:26.746094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519449003627293:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:26.746127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:27.290565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:27.290870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519449438096305:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-11T17:22:27.290900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519449438096305:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:27.290953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:27.290988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519449438096305:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:27.293207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:27.293462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519449003627293:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:27.293491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519449003627293:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:27.293562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:27.293612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519449003627293:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:27.305044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519449003627293:2048] [node 1] ICS08 incoming handshake Self# [1:7571519449438096303:0] Peer# [2:7571519449003627289:0] Counter# 2463 LastInputSerial# 61
2025-11-11T17:22:27.308594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519449438096305:2048] [node 2] ICS08 incoming handshake Self# [2:7571519449003627289:0] Peer# [1:7571519449438096303:0] Counter# 61 LastInputSerial# 2463
2025-11-11T17:22:27.311192Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519453733063603:2224] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:27.311537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:27.311590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519449438096305:2048] [node 2] ICS09 handshake done sender: [1:7571519453733063603:2224] self: [1:7571519449438096303:0] peer: [2:7571519449003627289:0] socket: 20 qp: 3527
2025-11-11T17:22:27.312122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519449438096305:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:27.312160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:27.314172Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519453298594592:2261] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:27.314451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:27.314527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519449003627293:2048] [node 1] ICS09 handshake done sender: [2:7571519453298594592:2261] self: [2:7571519449003627289:0] peer: [1:7571519449438096303:0] socket: 21 qp: 3526
2025-11-11T17:22:27.315552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519449003627293:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:27.315607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:27.630063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:27.630420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519449003627293:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-11-11T17:22:27.630456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519449003627293:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:27.630508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:27.630541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519449003627293:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:27.631662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:27.631978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519449438096305:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:27.632004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519449438096305:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:27.632059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:27.632095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519449438096305:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:27.637011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519449438096305:2048] [node 2] ICS08 incoming handshake Self# [2:7571519449003627289:0] Peer# [1:7571519449438096303:0] Counter# 116 LastInputSerial# 4644
2025-11-11T17:22:27.637011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519449003627293:2048] [node 1] ICS08 incoming handshake Self# [1:7571519449438096303:0] Peer# [2:7571519449003627289:0] Counter# 4644 LastInputSerial# 116
2025-11-11T17:22:27.637616Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519453733063608:2226] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:27.637906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:27.637975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519449438096305:2048] [node 2] ICS09 handshake done sender: [1:7571519453733063608:2226] self: [1:7571519449438096303:0] peer: [2:7571519449003627289:0] socket: 18 qp: 3532
2025-11-11T17:22:27.638509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519449438096305:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:27.638556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:27.642638Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:27.644202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519453298594598:2264] held: no
2025-11-11T17:22:27.644289Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519453298594598:2264] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:27.644531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:27.644588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519449003627293:2048] [node 1] ICS09 handshake done sender: [2:7571519453298594598:2264] self: [2:7571519449003627289:0] peer: [1:7571519449438096303:0] socket: 19 qp: 3533
2025-11-11T17:22:27.645270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519449003627293:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:27.645320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:27.817826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:27.818110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519449438096305:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:27.818134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519449438096305:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:27.818210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:27.818239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519449438096305:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:27.819850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:27.819893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519449003627293:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:27.819925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519449003627293:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:27.819953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:27.823194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:27.823484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519449003627293:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:27.823506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519449003627293:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:27.826045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519449003627293:2048] [node 1] ICS08 incoming handshake Self# [1:7571519449438096303:0] Peer# [2:7571519449003627289:0] Counter# 5985 LastInputSerial# 152
2025-11-11T17:22:27.829052Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519453733063613:2228] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:27.829421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:27.829483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519449438096305:2048] [node 2] ICS09 handshake done sender: [1:7571519453733063613:2228] self: [1:7571519449438096303:0] peer: [2:7571519449003627289:0] socket: 16 qp: 3534
2025-11-11T17:22:27.829961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519449438096305:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:27.830010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:27.833666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519449438096305:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:27.833705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519449438096305:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:27.833758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:27.833803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519449438096305:2048] [node 2] ICS08 incoming handshake Self# [2:7571519449003627289:0] Peer# [1:7571519449438096303:0] Counter# 152 LastInputSerial# 5985
2025-11-11T17:22:27.834081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:27.834331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519449438096305:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:27.834364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519449438096305:2048] VirtualId# [1:7571519449438096303:0]
2025-11-11T17:22:27.834404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:27.836152Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519453298594602:2267] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:27.836549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:27.836637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519449003627293:2048] [node 1] ICS09 handshake done sender: [2:7571519453298594602:2267] self: [2:7571519449003627289:0] peer: [1:7571519449438096303:0] socket: 17 qp: 3535
2025-11-11T17:22:27.837283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519449003627293:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:27.837323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:27.837625Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:27.837649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519449003627293:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:27.837673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519449003627293:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:27.837695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:27.837764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:27.837964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519449003627293:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:27.837982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519449003627293:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:27.841625Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:27.846025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519453733063619:2231]) peer tries to resume nonexistent session Self# [2:7571519449003627289:0] Peer# [1:7571519449438096303:0]
2025-11-11T17:22:27.846084Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519453733063619:2231] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:27.847000Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519453298594607:2269] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:27.848011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:27.848089Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:27.848118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519449003627293:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:27.848152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519449003627293:2048] VirtualId# [2:7571519449003627289:0]
2025-11-11T17:22:27.848303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 34637
TEvUndelivered Cookie# 34638
TEvUndelivered Cookie# 34639
TEvUndelivered Cookie# 34640
TEvUndelivered Cookie# 34641
TEvUndelivered Cookie# 34642
TEvUndelivered Cookie# 34643
2025-11-11T17:22:27.853337Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519453733063620:2232] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:27.853487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:27.854717Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519453298594610:2270] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:27.857544Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519453733063620:2232] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:27.857752Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519453298594610:2270] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:27.858013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:27.858080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:27.858128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519453298594613:2048]
2025-11-11T17:22:27.858172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519453298594613:2048] [node 1] ICS09 handshake done sender: [2:7571519453298594610:2270] self: [2:7571519453298594609:0] peer: [1:7571519453733063621:0] socket: 16 qp: 3540
2025-11-11T17:22:27.858205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519453298594613:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:27.858242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:27.858784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:27.858823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:27.858866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519453733063623:2048]
2025-11-11T17:22:27.858908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519453733063623:2048] [node 2] ICS09 handshake done sender: [1:7571519453733063620:2232] self: [1:7571519453733063621:0] peer: [2:7571519453298594609:0] socket: 17 qp: 3539
2025-11-11T17:22:27.858938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519453733063623:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:27.858980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:28.388222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519453298594613:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:28.388282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519453298594613:2048] VirtualId# [2:7571519453298594609:0]
2025-11-11T17:22:28.388321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519453298594613:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:28.388377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:28.388495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 35342
TEvUndelivered Cookie# 35343
TEvUndelivered Cookie# 35344
TEvUndelivered Cookie# 35345
TEvUndelivered Cookie# 35346
TEvUndelivered Cookie# 35347
TEvUndelivered Cookie# 35348
TEvUndelivered Cookie# 35349
2025-11-11T17:22:28.390108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:28.390446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519453733063623:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:28.390480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519453733063623:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:28.390557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:28.390604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519453733063623:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:28.393563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519457593561911:2271]) peer tries to resume nonexistent session Self# [1:7571519453733063621:0] Peer# [2:7571519453298594609:0]
2025-11-11T17:22:28.393652Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519457593561911:2271] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:28.393808Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519458028030921:2234] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:28.396251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:28.396354Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:28.396386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519453733063623:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:28.396425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519453733063623:2048] VirtualId# [1:7571519453733063621:0]
2025-11-11T17:22:28.396508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:28.400031Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519457593561912:2272] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:28.400244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:28.406172Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519458028030924:2235] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:28.409819Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519457593561912:2272] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:28.410063Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519458028030924:2235] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:28.410317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:28.410385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:28.410434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519457593561915:2048]
2025-11-11T17:22:28.410476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519457593561915:2048] [node 1] ICS09 handshake done sender: [2:7571519457593561912:2272] self: [2:7571519457593561913:0] peer: [1:7571519458028030923:0] socket: 17 qp: 3543
2025-11-11T17:22:28.410511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519457593561915:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:28.410419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:28.410552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:28.410464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:28.410508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519458028030927:2048]
2025-11-11T17:22:28.410545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519458028030927:2048] [node 2] ICS09 handshake done sender: [1:7571519458028030924:2235] self: [1:7571519458028030923:0] peer: [2:7571519457593561913:0] socket: 16 qp: 3542
2025-11-11T17:22:28.410572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519458028030927:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:28.410623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:28.801195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:28.801577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519458028030927:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:28.801649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519458028030927:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:28.801703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:28.801738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519458028030927:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:28.798723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:28.800810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:28.800844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519457593561915:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:28.800876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519457593561915:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:28.800914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:28.805066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:28.805345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519457593561915:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:28.805369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519457593561915:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:28.808578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519457593561915:2048] [node 1] ICS08 incoming handshake Self# [1:7571519458028030923:0] Peer# [2:7571519457593561913:0] Counter# 1955 LastInputSerial# 50
2025-11-11T17:22:28.811693Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519458028030929:2236] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:28.811980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:28.812041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519458028030927:2048] [node 2] ICS09 handshake done sender: [1:7571519458028030929:2236] self: [1:7571519458028030923:0] peer: [2:7571519457593561913:0] socket: 20 qp: 3544
2025-11-11T17:22:28.812748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519458028030927:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:28.812800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:28.824695Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519457593561918:2275] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:28.822121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519458028030927:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:28.822164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519458028030927:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:28.822244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:28.822295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519458028030927:2048] [node 2] ICS08 incoming handshake Self# [2:7571519457593561913:0] Peer# [1:7571519458028030923:0] Counter# 50 LastInputSerial# 1955
2025-11-11T17:22:28.822450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:28.822718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519458028030927:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:28.822752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519458028030927:2048] VirtualId# [1:7571519458028030923:0]
2025-11-11T17:22:28.822786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:28.825194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:28.825256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519457593561915:2048] [node 1] ICS09 handshake done sender: [2:7571519457593561918:2275] self: [2:7571519457593561913:0] peer: [1:7571519458028030923:0] socket: 21 qp: 3545
2025-11-11T17:22:28.825932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519457593561915:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:28.825980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:28.826247Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:28.826273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519457593561915:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:28.826299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519457593561915:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:28.826322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:28.826394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:28.826611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519457593561915:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:28.826630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519457593561915:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:28.830342Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:28.835225Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519458028030935:2239]) peer tries to resume nonexistent session Self# [2:7571519457593561913:0] Peer# [1:7571519458028030923:0]
2025-11-11T17:22:28.835297Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519458028030935:2239] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:28.836253Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519457593561923:2277] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:28.837194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:28.837275Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:28.837304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519457593561915:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:28.837338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519457593561915:2048] VirtualId# [2:7571519457593561913:0]
2025-11-11T17:22:28.837487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 35852
TEvUndelivered Cookie# 35853
TEvUndelivered Cookie# 35854
TEvUndelivered Cookie# 35855
TEvUndelivered Cookie# 35856
TEvUndelivered Cookie# 35857
TEvUndelivered Cookie# 35858
TEvUndelivered Cookie# 35859
2025-11-11T17:22:28.842920Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519458028030936:2240] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:28.843067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:28.850138Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519457593561926:2278] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:28.854589Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519458028030936:2240] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:28.857863Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519457593561926:2278] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:28.858239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:28.858295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:28.858345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519457593561929:2048]
2025-11-11T17:22:28.858392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519457593561929:2048] [node 1] ICS09 handshake done sender: [2:7571519457593561926:2278] self: [2:7571519457593561925:0] peer: [1:7571519458028030937:0] socket: 16 qp: 3550
2025-11-11T17:22:28.858427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519457593561929:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:28.858470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:28.861886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:28.861947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:28.862012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519458028030939:2048]
2025-11-11T17:22:28.862058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519458028030939:2048] [node 2] ICS09 handshake done sender: [1:7571519458028030936:2240] self: [1:7571519458028030937:0] peer: [2:7571519457593561925:0] socket: 17 qp: 3549
2025-11-11T17:22:28.862096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519458028030939:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:28.862129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:28.945700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519458028030939:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-11T17:22:28.945756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519458028030939:2048] VirtualId# [1:7571519458028030937:0]
2025-11-11T17:22:28.945795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519458028030939:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:28.945897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:28.945932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:28.949870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:28.950172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519457593561929:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:28.950205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519457593561929:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:28.950247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:28.950326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519457593561929:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:28.955933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519458028030941:2242]) peer tries to resume nonexistent session Self# [2:7571519457593561925:0] Peer# [1:7571519458028030937:0]
2025-11-11T17:22:28.956004Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519458028030941:2242] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:28.957783Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519457593561931:2279] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:28.961978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:28.962069Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:28.962095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519457593561929:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:28.962126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519457593561929:2048] VirtualId# [2:7571519457593561925:0]
2025-11-11T17:22:28.962178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:28.967976Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519458028030942:2243] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:28.968124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:28.968508Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519457593561934:2280] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:28.981838Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519458028030942:2243] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:28.982181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:28.982228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:28.982273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519458028030945:2048]
2025-11-11T17:22:28.982311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519458028030945:2048] [node 2] ICS09 handshake done sender: [1:7571519458028030942:2243] self: [1:7571519458028030943:0] peer: [2:7571519457593561933:0] socket: 17 qp: 3552
2025-11-11T17:22:28.982344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519458028030945:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:28.982377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:28.985750Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519457593561934:2280] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:28.990009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:28.990065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:28.990109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519457593561937:2048]
2025-11-11T17:22:28.990149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519457593561937:2048] [node 1] ICS09 handshake done sender: [2:7571519457593561934:2280] self: [2:7571519457593561933:0] peer: [1:7571519458028030943:0] socket: 16 qp: 3553
2025-11-11T17:22:28.990181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519457593561937:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:28.990215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:29.253933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:29.254186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519457593561937:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:29.254211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519457593561937:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:29.254273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:29.254304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519457593561937:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:29.254947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:29.255185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519458028030945:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:29.255205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519458028030945:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:29.255269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:29.255315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519458028030945:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:29.260734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519457593561937:2048] [node 1] ICS08 incoming handshake Self# [1:7571519458028030943:0] Peer# [2:7571519457593561933:0] Counter# 1107 LastInputSerial# 27
2025-11-11T17:22:29.269860Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519462322998243:2245] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:29.270133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:29.270179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519458028030945:2048] [node 2] ICS09 handshake done sender: [1:7571519462322998243:2245] self: [1:7571519458028030943:0] peer: [2:7571519457593561933:0] socket: 20 qp: 3554
2025-11-11T17:22:29.270707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519458028030945:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:29.270754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:29.273916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519458028030945:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:29.273940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519458028030945:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:29.273990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:29.274030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519458028030945:2048] [node 2] ICS08 incoming handshake Self# [2:7571519457593561933:0] Peer# [1:7571519458028030943:0] Counter# 27 LastInputSerial# 1107
2025-11-11T17:22:29.274160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:29.274350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519458028030945:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:29.274379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519458028030945:2048] VirtualId# [1:7571519458028030943:0]
2025-11-11T17:22:29.274407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:29.283030Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519461888529236:2282] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:29.283638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:29.283687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519457593561937:2048] [node 1] ICS09 handshake done sender: [2:7571519461888529236:2282] self: [2:7571519457593561933:0] peer: [1:7571519458028030943:0] socket: 21 qp: 3555
2025-11-11T17:22:29.284208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519457593561937:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:29.284247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:29.284471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:29.284491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519457593561937:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:29.284512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519457593561937:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:29.284535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:29.284603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:29.284768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519457593561937:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:29.284784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519457593561937:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:29.287930Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:29.292369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519462322998249:2248]) peer tries to resume nonexistent session Self# [2:7571519457593561933:0] Peer# [1:7571519458028030943:0]
2025-11-11T17:22:29.292418Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519462322998249:2248] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:29.293700Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519461888529241:2284] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:29.297920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:29.297998Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:29.298022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519457593561937:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:29.298052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519457593561937:2048] VirtualId# [2:7571519457593561933:0]
2025-11-11T17:22:29.298183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 36137
TEvUndelivered Cookie# 36138
TEvUndelivered Cookie# 36139
TEvUndelivered Cookie# 36140
TEvUndelivered Cookie# 36141
TEvUndelivered Cookie# 36142
TEvUndelivered Cookie# 36143
TEvUndelivered Cookie# 36144
2025-11-11T17:22:29.310032Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519462322998250:2249] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:29.310150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:29.318095Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519461888529244:2285] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:29.325819Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519462322998250:2249] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:29.326376Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519461888529244:2285] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:29.326637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:29.326680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:29.326719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519461888529247:2048]
2025-11-11T17:22:29.326757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519461888529247:2048] [node 1] ICS09 handshake done sender: [2:7571519461888529244:2285] self: [2:7571519461888529243:0] peer: [1:7571519462322998251:0] socket: 16 qp: 3560
2025-11-11T17:22:29.326787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519461888529247:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:29.326818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:29.327299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:29.327333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:29.327366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519462322998253:2048]
2025-11-11T17:22:29.327397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519462322998253:2048] [node 2] ICS09 handshake done sender: [1:7571519462322998250:2249] self: [1:7571519462322998251:0] peer: [2:7571519461888529243:0] socket: 17 qp: 3559
2025-11-11T17:22:29.327420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519462322998253:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:29.327445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:29.546355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519462322998253:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-11T17:22:29.546410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519462322998253:2048] VirtualId# [1:7571519462322998251:0]
2025-11-11T17:22:29.546442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519462322998253:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:29.546499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:29.546544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:29.548843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:29.549243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519461888529247:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:29.549267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519461888529247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:29.549298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:29.549337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519461888529247:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:29.555323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519462322998255:2251]) peer tries to resume nonexistent session Self# [2:7571519461888529243:0] Peer# [1:7571519462322998251:0]
2025-11-11T17:22:29.555381Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519462322998255:2251] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:29.556257Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519461888529249:2286] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:29.562050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:29.562136Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:29.562162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519461888529247:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:29.562192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519461888529247:2048] VirtualId# [2:7571519461888529243:0]
2025-11-11T17:22:29.562479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 36308
TEvUndelivered Cookie# 36309
TEvUndelivered Cookie# 36310
TEvUndelivered Cookie# 36311
TEvUndelivered Cookie# 36312
TEvUndelivered Cookie# 36313
TEvUndelivered Cookie# 36314
TEvUndelivered Cookie# 36315
TEvUndelivered Cookie# 36316
2025-11-11T17:22:29.567544Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519462322998256:2252] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:29.567682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:29.568075Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519461888529252:2287] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:29.578041Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519462322998256:2252] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:29.578477Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519461888529252:2287] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:29.578838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:29.578909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:29.578950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519461888529255:2048]
2025-11-11T17:22:29.578987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519461888529255:2048] [node 1] ICS09 handshake done sender: [2:7571519461888529252:2287] self: [2:7571519461888529251:0] peer: [1:7571519462322998257:0] socket: 16 qp: 3563
2025-11-11T17:22:29.579019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519461888529255:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:29.579050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:29.579640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:29.579679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:29.579715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519462322998259:2048]
2025-11-11T17:22:29.579749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519462322998259:2048] [node 2] ICS09 handshake done sender: [1:7571519462322998256:2252] self: [1:7571519462322998257:0] peer: [2:7571519461888529251:0] socket: 17 qp: 3562
2025-11-11T17:22:29.579772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519462322998259:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:29.579796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
TEvNodeDisconnected
TEvUndelivered Cookie# 36636
TEvUndelivered Cookie# 36637
TEvUndelivered Cookie# 36638
TEvUndelivered Cookie# 36639
TEvUndelivered Cookie# 36640
TEvUndelivered Cookie# 36641
TEvUndelivered Cookie# 36642
TEvUndelivered Cookie# 36643
2025-11-11T17:22:29.858091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519461888529255:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:29.858133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519461888529255:2048] VirtualId# [2:7571519461888529251:0]
2025-11-11T17:22:29.858159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519461888529255:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:29.858210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:29.858301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:29.859085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:29.859335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519462322998259:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:29.859358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519462322998259:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:29.859429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:29.859458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519462322998259:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:29.863442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519461888529257:2288]) peer tries to resume nonexistent session Self# [1:7571519462322998257:0] Peer# [2:7571519461888529251:0]
2025-11-11T17:22:29.863508Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519461888529257:2288] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:29.864635Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519462322998261:2254] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:29.865624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:29.865690Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:29.865713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519462322998259:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:29.865741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519462322998259:2048] VirtualId# [1:7571519462322998257:0]
2025-11-11T17:22:29.865786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:29.873527Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519462322998263:2255] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:29.873865Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519461888529261:2290] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:29.873960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:29.874318Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519462322998265:2256] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:29.881849Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519461888529261:2290] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:29.882104Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519462322998265:2256] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:29.882333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:29.882394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:29.882439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519462322998269:2048]
2025-11-11T17:22:29.882478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519462322998269:2048] [node 2] ICS09 handshake done sender: [1:7571519462322998265:2256] self: [1:7571519462322998264:0] peer: [2:7571519461888529262:0] socket: 17 qp: 3568
2025-11-11T17:22:29.882510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519462322998269:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:29.882543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:29.883907Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:29.884984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:29.885035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:29.885077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519461888529264:2048]
2025-11-11T17:22:29.885112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519461888529264:2048] [node 1] ICS09 handshake done sender: [2:7571519461888529261:2290] self: [2:7571519461888529262:0] peer: [1:7571519462322998264:0] socket: 16 qp: 3567
2025-11-11T17:22:29.885142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519461888529264:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:29.885175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:29.885356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:30.200884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:30.201001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [1 <-> 2] socket: Broken pipe
2025-11-11T17:22:30.201025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519462322998269:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:30.201048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519462322998269:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:30.201072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:30.206139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:30.206379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519461888529264:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:30.206402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519461888529264:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:30.206493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:30.206526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519461888529264:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:30.208812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:30.209077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519462322998269:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:30.209095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519462322998269:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:30.213016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519462322998269:2048] [node 2] ICS08 incoming handshake Self# [2:7571519461888529262:0] Peer# [1:7571519462322998264:0] Counter# 41 LastInputSerial# 1618
2025-11-11T17:22:30.214773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519461888529264:2048] [node 1] ICS08 incoming handshake Self# [1:7571519462322998264:0] Peer# [2:7571519461888529262:0] Counter# 1618 LastInputSerial# 41
2025-11-11T17:22:30.221938Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519466617965567:2257] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:30.222385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:30.222447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519462322998269:2048] [node 2] ICS09 handshake done sender: [1:7571519466617965567:2257] self: [1:7571519462322998264:0] peer: [2:7571519461888529262:0] socket: 22 qp: 3572
2025-11-11T17:22:30.222829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519462322998269:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:30.222872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:30.224737Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519466183496564:2293] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:30.224953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:30.225003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519461888529264:2048] [node 1] ICS09 handshake done sender: [2:7571519466183496564:2293] self: [2:7571519461888529262:0] peer: [1:7571519462322998264:0] socket: 23 qp: 3571
2025-11-11T17:22:30.225668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519461888529264:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:30.225737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:30.629830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:30.629986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:30.630008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519461888529264:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:30.630029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519461888529264:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:30.630049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:30.630325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:30.630584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519462322998269:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:30.630608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519462322998269:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:30.630667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:30.630696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519462322998269:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:30.632748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:30.632951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519461888529264:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:30.632969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519461888529264:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:30.634779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519461888529264:2048] [node 1] ICS08 incoming handshake Self# [1:7571519462322998264:0] Peer# [2:7571519461888529262:0] Counter# 5293 LastInputSerial# 134
2025-11-11T17:22:30.639388Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519466617965572:2259] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:30.639741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:30.639792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519462322998269:2048] [node 2] ICS09 handshake done sender: [1:7571519466617965572:2259] self: [1:7571519462322998264:0] peer: [2:7571519461888529262:0] socket: 16 qp: 3573
2025-11-11T17:22:30.640226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519462322998269:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:30.640266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:30.640358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519462322998269:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:30.640380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519462322998269:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:30.640572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:30.640618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519462322998269:2048] [node 2] ICS08 incoming handshake Self# [2:7571519461888529262:0] Peer# [1:7571519462322998264:0] Counter# 134 LastInputSerial# 5293
2025-11-11T17:22:30.640690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:30.640877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519462322998269:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:30.640904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519462322998269:2048] VirtualId# [1:7571519462322998264:0]
2025-11-11T17:22:30.640930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:30.643093Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519466183496568:2296] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:30.643447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:30.643498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519461888529264:2048] [node 1] ICS09 handshake done sender: [2:7571519466183496568:2296] self: [2:7571519461888529262:0] peer: [1:7571519462322998264:0] socket: 17 qp: 3574
2025-11-11T17:22:30.644117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519461888529264:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:30.644156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:30.644410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:30.644431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519461888529264:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:30.644451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519461888529264:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:30.644470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:30.646502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:30.646693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519461888529264:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:30.646710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519461888529264:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:30.647682Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:30.652975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519466617965578:2262]) peer tries to resume nonexistent session Self# [2:7571519461888529262:0] Peer# [1:7571519462322998264:0]
2025-11-11T17:22:30.653026Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519466617965578:2262] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:30.653683Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519466183496573:2298] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:30.654824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:30.654900Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:30.654928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519461888529264:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:30.654959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519461888529264:2048] VirtualId# [2:7571519461888529262:0]
2025-11-11T17:22:30.655065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 37931
TEvUndelivered Cookie# 37932
TEvUndelivered Cookie# 37933
TEvUndelivered Cookie# 37934
TEvUndelivered Cookie# 37935
TEvUndelivered Cookie# 37936
2025-11-11T17:22:30.659883Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519466617965579:2263] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:30.660001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:30.661014Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519466183496576:2299] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:30.665967Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519466617965579:2263] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:30.669436Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519466183496576:2299] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:30.669837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:30.669887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:30.669931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519466617965582:2048]
2025-11-11T17:22:30.669970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519466617965582:2048] [node 2] ICS09 handshake done sender: [1:7571519466617965579:2263] self: [1:7571519466617965580:0] peer: [2:7571519466183496575:0] socket: 17 qp: 3578
2025-11-11T17:22:30.670002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519466617965582:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:30.670036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:30.670592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:30.670629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:30.670660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519466183496579:2048]
2025-11-11T17:22:30.670693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519466183496579:2048] [node 1] ICS09 handshake done sender: [2:7571519466183496576:2299] self: [2:7571519466183496575:0] peer: [1:7571519466617965580:0] socket: 16 qp: 3579
2025-11-11T17:22:30.670713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519466183496579:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:30.670737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:30.945988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:30.946677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:30.946924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519466617965582:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:30.946949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519466617965582:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:30.947163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:30.947208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519466617965582:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:30.950513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:30.950734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519466183496579:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:30.950758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519466183496579:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:30.950822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:30.950856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519466183496579:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:30.953362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519466183496579:2048] [node 1] ICS08 incoming handshake Self# [1:7571519466617965580:0] Peer# [2:7571519466183496575:0] Counter# 1640 LastInputSerial# 38
2025-11-11T17:22:30.958753Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519466617965584:2265] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:30.959354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:30.959400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519466617965582:2048] [node 2] ICS09 handshake done sender: [1:7571519466617965584:2265] self: [1:7571519466617965580:0] peer: [2:7571519466183496575:0] socket: 20 qp: 3580
2025-11-11T17:22:30.959765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519466617965582:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:30.959805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:30.959983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519466617965582:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:30.960002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519466617965582:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:30.960030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:30.960065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519466617965582:2048] [node 2] ICS08 incoming handshake Self# [2:7571519466183496575:0] Peer# [1:7571519466617965580:0] Counter# 38 LastInputSerial# 1640
2025-11-11T17:22:30.960128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:30.960309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519466617965582:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:30.961639Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519466183496582:2301] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:30.961962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:30.962008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519466183496579:2048] [node 1] ICS09 handshake done sender: [2:7571519466183496582:2301] self: [2:7571519466183496575:0] peer: [1:7571519466617965580:0] socket: 21 qp: 3581
2025-11-11T17:22:30.962657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519466183496579:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:30.962716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
TEvNodeDisconnected
2025-11-11T17:22:30.972633Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:30.968806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:30.969010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519466183496579:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-11T17:22:30.969039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519466183496579:2048] VirtualId# [2:7571519466183496575:0]
2025-11-11T17:22:30.969062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519466183496579:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:30.969124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:30.969164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:30.975402Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519466617965590:2268] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:30.975516Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:30.975538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519466617965582:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:30.975565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519466617965582:2048] VirtualId# [1:7571519466617965580:0]
2025-11-11T17:22:30.975609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:30.975652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:30.977296Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519466183496588:2303] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:30.980108Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519466617965590:2268] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:30.980707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:30.980754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:30.980797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519466617965593:2048]
2025-11-11T17:22:30.980836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519466617965593:2048] [node 2] ICS09 handshake done sender: [1:7571519466617965590:2268] self: [1:7571519466617965591:0] peer: [2:7571519466183496587:0] socket: 18 qp: 3585
2025-11-11T17:22:30.980868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519466617965593:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:30.980899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:30.980230Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519466183496588:2303] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:30.981367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:30.981399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:30.981430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519466183496591:2048]
2025-11-11T17:22:30.981462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519466183496591:2048] [node 1] ICS09 handshake done sender: [2:7571519466183496588:2303] self: [2:7571519466183496587:0] peer: [1:7571519466617965591:0] socket: 16 qp: 3584
2025-11-11T17:22:30.981487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519466183496591:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:30.981509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:31.406101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519466183496591:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-11T17:22:31.406150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519466183496591:2048] VirtualId# [2:7571519466183496587:0]
2025-11-11T17:22:31.408055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:31.408332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519466617965593:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:31.408360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519466617965593:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:31.408426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:31.408459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519466617965593:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:31.406178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519466183496591:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:31.406221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:31.406299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 38949
TEvUndelivered Cookie# 38950
TEvUndelivered Cookie# 38951
TEvUndelivered Cookie# 38952
TEvUndelivered Cookie# 38953
2025-11-11T17:22:31.413972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519470478463889:2304]) peer tries to resume nonexistent session Self# [1:7571519466617965591:0] Peer# [2:7571519466183496587:0]
2025-11-11T17:22:31.414037Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519470478463889:2304] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:31.414815Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519470912932891:2270] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:31.416135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:31.416206Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:31.416232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519466617965593:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:31.416260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519466617965593:2048] VirtualId# [1:7571519466617965591:0]
2025-11-11T17:22:31.416331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:31.422567Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519470478463890:2305] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:31.422721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:31.423125Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519470912932894:2271] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:31.427479Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519470478463890:2305] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:31.427551Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519470912932894:2271] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:31.427846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:31.427897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:31.427948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519470912932897:2048]
2025-11-11T17:22:31.427986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519470912932897:2048] [node 2] ICS09 handshake done sender: [1:7571519470912932894:2271] self: [1:7571519470912932893:0] peer: [2:7571519470478463891:0] socket: 16 qp: 3588
2025-11-11T17:22:31.428013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519470912932897:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:31.428044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:31.427986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:31.428036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:31.428073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519470478463893:2048]
2025-11-11T17:22:31.428108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519470478463893:2048] [node 1] ICS09 handshake done sender: [2:7571519470478463890:2305] self: [2:7571519470478463891:0] peer: [1:7571519470912932893:0] socket: 17 qp: 3587
2025-11-11T17:22:31.428134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519470478463893:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:31.428162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:31.658747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:31.659088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519470912932897:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:31.659111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519470912932897:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:31.659153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:31.659182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519470912932897:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:31.660922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:31.661137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519470478463893:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:31.661158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519470478463893:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:31.661221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:31.661252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519470478463893:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:31.666925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519470478463893:2048] [node 1] ICS08 incoming handshake Self# [1:7571519470912932893:0] Peer# [2:7571519470478463891:0] Counter# 1358 LastInputSerial# 33
2025-11-11T17:22:31.668442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519470912932897:2048] [node 2] ICS08 incoming handshake Self# [2:7571519470478463891:0] Peer# [1:7571519470912932893:0] Counter# 33 LastInputSerial# 1358
2025-11-11T17:22:31.668812Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519470912932899:2272] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:31.669032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:31.669084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519470912932897:2048] [node 2] ICS09 handshake done sender: [1:7571519470912932899:2272] self: [1:7571519470912932893:0] peer: [2:7571519470478463891:0] socket: 20 qp: 3590
2025-11-11T17:22:31.669647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519470912932897:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:31.669684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:31.672765Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519470478463896:2308] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:31.673374Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:31.675028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:31.675085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519470478463893:2048] [node 1] ICS09 handshake done sender: [2:7571519470478463896:2308] self: [2:7571519470478463891:0] peer: [1:7571519470912932893:0] socket: 21 qp: 3589
2025-11-11T17:22:31.675690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519470478463893:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:31.675734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:31.675855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:31.885871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:31.886133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519470912932897:2048] [node 2] ICS07 socket disconnect 20 reason# Debug
2025-11-11T17:22:31.886157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519470912932897:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:31.886198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:31.886226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519470912932897:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:31.889681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:31.889936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519470478463893:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-11T17:22:31.889968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519470478463893:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:31.890035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:31.890073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519470478463893:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:31.892332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519470478463893:2048] [node 1] ICS08 incoming handshake Self# [1:7571519470912932893:0] Peer# [2:7571519470478463891:0] Counter# 2985 LastInputSerial# 75
2025-11-11T17:22:31.895985Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519470912932904:2274] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:31.896332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:31.896390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519470912932897:2048] [node 2] ICS09 handshake done sender: [1:7571519470912932904:2274] self: [1:7571519470912932893:0] peer: [2:7571519470478463891:0] socket: 16 qp: 3593
2025-11-11T17:22:31.896878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519470912932897:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:31.896917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:31.905920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519470912932897:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:31.905961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519470912932897:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:31.906035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:31.906074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519470912932897:2048] [node 2] ICS08 incoming handshake Self# [2:7571519470478463891:0] Peer# [1:7571519470912932893:0] Counter# 75 LastInputSerial# 2985
2025-11-11T17:22:31.906334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:31.906553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519470912932897:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:31.906582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519470912932897:2048] VirtualId# [1:7571519470912932893:0]
2025-11-11T17:22:31.906614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:31.910573Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519470478463901:2311] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:31.915672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:31.915737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519470478463893:2048] [node 1] ICS09 handshake done sender: [2:7571519470478463901:2311] self: [2:7571519470478463891:0] peer: [1:7571519470912932893:0] socket: 17 qp: 3594
2025-11-11T17:22:31.916519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519470478463893:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:31.916565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:31.916865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:31.916889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519470478463893:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:31.916914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519470478463893:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:31.916935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:31.917012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:31.917216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519470478463893:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:31.917235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519470478463893:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:31.927494Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:31.929990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519470912932910:2277]) peer tries to resume nonexistent session Self# [2:7571519470478463891:0] Peer# [1:7571519470912932893:0]
2025-11-11T17:22:31.930041Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519470912932910:2277] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:31.933837Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519470478463906:2313] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:31.934814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:31.934889Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:31.934915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519470478463893:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:31.934945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519470478463893:2048] VirtualId# [2:7571519470478463891:0]
2025-11-11T17:22:31.934998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:31.953873Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519470912932911:2278] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:31.954035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:31.969821Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519470478463909:2314] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:31.979891Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519470912932911:2278] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:31.981761Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519470478463909:2314] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:31.983980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:31.984037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:31.984085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519470912932914:2048]
2025-11-11T17:22:31.984126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519470912932914:2048] [node 2] ICS09 handshake done sender: [1:7571519470912932911:2278] self: [1:7571519470912932912:0] peer: [2:7571519470478463908:0] socket: 17 qp: 3600
2025-11-11T17:22:31.984159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519470912932914:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:31.984195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:31.993880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:31.993938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:31.993987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519470478463912:2048]
2025-11-11T17:22:31.994033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519470478463912:2048] [node 1] ICS09 handshake done sender: [2:7571519470478463909:2314] self: [2:7571519470478463908:0] peer: [1:7571519470912932912:0] socket: 16 qp: 3599
2025-11-11T17:22:31.994069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519470478463912:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:31.994101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:32.118163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:32.118411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519470912932914:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:32.118435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519470912932914:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:32.118495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:32.118525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519470912932914:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:32.117714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:32.117848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:32.117874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519470478463912:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:32.117897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519470478463912:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:32.117926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:32.125750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:32.125996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519470478463912:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:32.126017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519470478463912:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:32.128311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519470478463912:2048] [node 1] ICS08 incoming handshake Self# [1:7571519470912932912:0] Peer# [2:7571519470478463908:0] Counter# 421 LastInputSerial# 11
2025-11-11T17:22:32.130483Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519475207900212:2280] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:32.138045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:32.138097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519470912932914:2048] [node 2] ICS09 handshake done sender: [1:7571519475207900212:2280] self: [1:7571519470912932912:0] peer: [2:7571519470478463908:0] socket: 20 qp: 3601
2025-11-11T17:22:32.138591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519470912932914:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:32.138633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:32.138815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519470912932914:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:32.138838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519470912932914:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:32.138869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:32.138906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519470912932914:2048] [node 2] ICS08 incoming handshake Self# [2:7571519470478463908:0] Peer# [1:7571519470912932912:0] Counter# 11 LastInputSerial# 421
2025-11-11T17:22:32.138975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:32.139151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519470912932914:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:32.140681Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519474773431211:2316] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:32.141091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:32.141142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519470478463912:2048] [node 1] ICS09 handshake done sender: [2:7571519474773431211:2316] self: [2:7571519470478463908:0] peer: [1:7571519470912932912:0] socket: 21 qp: 3602
2025-11-11T17:22:32.141825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519470478463912:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:32.141863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:32.142165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:32.142186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519470478463912:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:32.142211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519470478463912:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:32.142230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:32.142348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:32.142506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519470478463912:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:32.142521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519470478463912:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:32.148521Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:32.152828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519470912932914:2048] [node 2] ICS08 incoming handshake Self# [2:7571519470478463908:0] Peer# [1:7571519470912932912:0] Counter# 12 LastInputSerial# 421
2025-11-11T17:22:32.160234Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519474773431216:2318] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:32.160551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:32.160601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519470478463912:2048] [node 1] ICS09 handshake done sender: [2:7571519474773431216:2318] self: [2:7571519470478463908:0] peer: [1:7571519470912932912:0] socket: 18 qp: 3606
2025-11-11T17:22:32.161063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519470478463912:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:32.161104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:32.162638Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519475207900218:2283] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:32.163122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:32.163168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519470912932914:2048] [node 2] ICS09 handshake done sender: [1:7571519475207900218:2283] self: [1:7571519470912932912:0] peer: [2:7571519470478463908:0] socket: 19 qp: 3605
2025-11-11T17:22:32.163824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519470912932914:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:32.163865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:32.642119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519470912932914:2048] [node 2] ICS01 socket: 19 reason# Debug
2025-11-11T17:22:32.642172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519470912932914:2048] VirtualId# [1:7571519470912932912:0]
2025-11-11T17:22:32.642206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519470912932914:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:32.642260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:32.642305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:32.645778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:32.646043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519470478463912:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:32.646068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519470478463912:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:32.646133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:32.646164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519470478463912:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:32.656637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519475207900221:2285]) peer tries to resume nonexistent session Self# [2:7571519470478463908:0] Peer# [1:7571519470912932912:0]
2025-11-11T17:22:32.658037Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519475207900221:2285] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:32.658879Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519474773431220:2319] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:32.660007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:32.660088Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:32.660114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519470478463912:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:32.660146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519470478463912:2048] VirtualId# [2:7571519470478463908:0]
2025-11-11T17:22:32.660200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:32.666158Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519474773431222:2320] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:32.666397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:32.666769Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519475207900225:2287] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:32.672565Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519475207900223:2286] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:32.685825Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519474773431222:2320] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:32.689994Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519475207900223:2286] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:32.690296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:32.690354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:32.690400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519475207900228:2048]
2025-11-11T17:22:32.690440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519475207900228:2048] [node 2] ICS09 handshake done sender: [1:7571519475207900223:2286] self: [1:7571519475207900222:0] peer: [2:7571519474773431226:0] socket: 19 qp: 3610
2025-11-11T17:22:32.690469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519475207900228:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:32.690499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:32.693838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:32.693899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:32.693949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519474773431228:2048]
2025-11-11T17:22:32.693992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519474773431228:2048] [node 1] ICS09 handshake done sender: [2:7571519474773431222:2320] self: [2:7571519474773431226:0] peer: [1:7571519475207900222:0] socket: 21 qp: 3609
2025-11-11T17:22:32.694025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519474773431228:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:32.694060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:32.694428Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8
2025-11-11T17:22:32.702135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:32.953782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519474773431228:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-11-11T17:22:32.953835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519474773431228:2048] VirtualId# [2:7571519474773431226:0]
2025-11-11T17:22:32.953869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519474773431228:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:32.953933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:32.953988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:32.954065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:32.954518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519475207900228:2048] [node 2] ICS01 socket: 19 reason# EndOfStream
2025-11-11T17:22:32.954562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519475207900228:2048] VirtualId# [1:7571519475207900222:0]
2025-11-11T17:22:32.954607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519475207900228:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:32.954639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:32.954686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:32.961369Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519474773431230:2323] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:32.961546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:32.961971Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519475207900231:2288] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:32.970913Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519474773431230:2323] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:32.973995Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519475207900231:2288] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:32.974215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:32.974278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:32.974324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519474773431233:2048]
2025-11-11T17:22:32.974365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519474773431233:2048] [node 1] ICS09 handshake done sender: [2:7571519474773431230:2323] self: [2:7571519474773431231:0] peer: [1:7571519475207900230:0] socket: 17 qp: 3613
2025-11-11T17:22:32.974397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519474773431233:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:32.974434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:32.975198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:32.975232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:32.975292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519475207900234:2048]
2025-11-11T17:22:32.975330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519475207900234:2048] [node 2] ICS09 handshake done sender: [1:7571519475207900231:2288] self: [1:7571519475207900230:0] peer: [2:7571519474773431231:0] socket: 16 qp: 3614
2025-11-11T17:22:32.975355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519475207900234:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:32.975380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:33.267598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:33.267898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519475207900234:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:33.267925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519475207900234:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:33.268001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:33.268040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519475207900234:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:33.268430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:33.268467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519474773431233:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:33.268492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519474773431233:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:33.268519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:33.270339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:33.270574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519474773431233:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:33.270592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519474773431233:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:33.272487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519474773431233:2048] [node 1] ICS08 incoming handshake Self# [1:7571519475207900230:0] Peer# [2:7571519474773431231:0] Counter# 1567 LastInputSerial# 40
2025-11-11T17:22:33.274742Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519479502867532:2289] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:33.275017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:33.275068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519475207900234:2048] [node 2] ICS09 handshake done sender: [1:7571519479502867532:2289] self: [1:7571519475207900230:0] peer: [2:7571519474773431231:0] socket: 20 qp: 3615
2025-11-11T17:22:33.275815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519475207900234:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:33.275858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:33.280564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519475207900234:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:33.280597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519475207900234:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:33.280649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:33.280693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519475207900234:2048] [node 2] ICS08 incoming handshake Self# [2:7571519474773431231:0] Peer# [1:7571519475207900230:0] Counter# 40 LastInputSerial# 1567
2025-11-11T17:22:33.280827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:33.281060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519475207900234:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:33.281098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519475207900234:2048] VirtualId# [1:7571519475207900230:0]
2025-11-11T17:22:33.281128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:33.283019Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519479068398532:2326] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:33.283453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:33.283512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519474773431233:2048] [node 1] ICS09 handshake done sender: [2:7571519479068398532:2326] self: [2:7571519474773431231:0] peer: [1:7571519475207900230:0] socket: 21 qp: 3616
2025-11-11T17:22:33.284017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519474773431233:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:33.284057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:33.284371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:33.284392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519474773431233:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:33.284416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519474773431233:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:33.284434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:33.284508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:33.284701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519474773431233:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:33.284718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519474773431233:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:33.288474Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:33.294164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519479502867538:2292]) peer tries to resume nonexistent session Self# [2:7571519474773431231:0] Peer# [1:7571519475207900230:0]
2025-11-11T17:22:33.294224Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519479502867538:2292] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:33.295096Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519479068398537:2328] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:33.296733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:33.296816Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:33.296845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519474773431233:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:33.296877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519474773431233:2048] VirtualId# [2:7571519474773431231:0]
2025-11-11T17:22:33.296999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 41360
TEvUndelivered Cookie# 41361
TEvUndelivered Cookie# 41362
TEvUndelivered Cookie# 41363
TEvUndelivered Cookie# 41364
TEvUndelivered Cookie# 41365
2025-11-11T17:22:33.310114Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519479502867539:2293] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:33.310266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:33.312561Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519479068398540:2329] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:33.325843Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519479502867539:2293] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:33.326014Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519479068398540:2329] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:33.326268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:33.326311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:33.326355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519479068398543:2048]
2025-11-11T17:22:33.326395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519479068398543:2048] [node 1] ICS09 handshake done sender: [2:7571519479068398540:2329] self: [2:7571519479068398539:0] peer: [1:7571519479502867540:0] socket: 16 qp: 3621
2025-11-11T17:22:33.326428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519479068398543:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:33.326461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:33.330146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:33.330198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:33.330264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519479502867542:2048]
2025-11-11T17:22:33.330304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519479502867542:2048] [node 2] ICS09 handshake done sender: [1:7571519479502867539:2293] self: [1:7571519479502867540:0] peer: [2:7571519479068398539:0] socket: 17 qp: 3620
2025-11-11T17:22:33.330338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519479502867542:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:33.330372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:33.441864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:33.442156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519479068398543:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:33.442180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519479068398543:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:33.442239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:33.442267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519479068398543:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:33.445862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:33.446130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519479502867542:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:33.446155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519479502867542:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:33.446221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:33.446258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519479502867542:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:33.452550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519479068398543:2048] [node 1] ICS08 incoming handshake Self# [1:7571519479502867540:0] Peer# [2:7571519479068398539:0] Counter# 229 LastInputSerial# 5
2025-11-11T17:22:33.453848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519479502867542:2048] [node 2] ICS08 incoming handshake Self# [2:7571519479068398539:0] Peer# [1:7571519479502867540:0] Counter# 5 LastInputSerial# 229
2025-11-11T17:22:33.454259Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519479502867544:2295] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:33.454526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:33.454581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519479502867542:2048] [node 2] ICS09 handshake done sender: [1:7571519479502867544:2295] self: [1:7571519479502867540:0] peer: [2:7571519479068398539:0] socket: 22 qp: 3623
2025-11-11T17:22:33.455235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519479502867542:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:33.455296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:33.457822Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:33.462080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519479068398547:2331] held: no
2025-11-11T17:22:33.464579Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519479068398547:2331] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:33.464781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:33.464836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519479068398543:2048] [node 1] ICS09 handshake done sender: [2:7571519479068398547:2331] self: [2:7571519479068398539:0] peer: [1:7571519479502867540:0] socket: 23 qp: 3625
2025-11-11T17:22:33.465514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519479068398543:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:33.465560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:33.878237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:33.878490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519479502867542:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:33.878513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519479502867542:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:33.878578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:33.878607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519479502867542:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:33.877774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:33.877911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:33.877934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519479068398543:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:33.877957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519479068398543:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:33.877981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:33.880730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:33.880915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519479068398543:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:33.880934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519479068398543:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:33.883046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519479068398543:2048] [node 1] ICS08 incoming handshake Self# [1:7571519479502867540:0] Peer# [2:7571519479068398539:0] Counter# 3313 LastInputSerial# 84
2025-11-11T17:22:33.890078Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519479502867549:2297] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:33.891152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:33.891207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519479502867542:2048] [node 2] ICS09 handshake done sender: [1:7571519479502867549:2297] self: [1:7571519479502867540:0] peer: [2:7571519479068398539:0] socket: 16 qp: 3626
2025-11-11T17:22:33.891686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519479502867542:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:33.891726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:33.895518Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519479068398551:2334] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:33.895842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:33.895902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519479068398543:2048] [node 1] ICS09 handshake done sender: [2:7571519479068398551:2334] self: [2:7571519479068398539:0] peer: [1:7571519479502867540:0] socket: 18 qp: 3627
2025-11-11T17:22:33.896297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519479068398543:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:33.896343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:33.902070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519479502867542:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:33.902107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519479502867542:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:33.902156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:33.902200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519479502867542:2048] [node 2] ICS08 incoming handshake Self# [2:7571519479068398539:0] Peer# [1:7571519479502867540:0] Counter# 84 LastInputSerial# 3350
2025-11-11T17:22:33.902609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:33.902829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519479068398543:2048] [node 1] ICS01 socket: 18 reason# EndOfStream
2025-11-11T17:22:33.902861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519479068398543:2048] VirtualId# [2:7571519479068398539:0]
2025-11-11T17:22:33.902886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519479068398543:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:33.902940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:33.902975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:33.904038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:33.904248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519479502867542:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
Subscribe
2025-11-11T17:22:34.011999Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519483797834850:2299] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:34.012120Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:34.012143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519479502867542:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:34.012172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519479502867542:2048] VirtualId# [1:7571519479502867540:0]
2025-11-11T17:22:34.012256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:34.012292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:34.018038Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519483363365852:2336] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:34.024623Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519483797834850:2299] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:34.025922Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519483363365852:2336] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-11T17:22:34.026245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:34.026313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:34.026360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519483363365855:2048]
2025-11-11T17:22:34.026401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519483363365855:2048] [node 1] ICS09 handshake done sender: [2:7571519483363365852:2336] self: [2:7571519483363365851:0] peer: [1:7571519483797834851:0] socket: 17 qp: 3630
2025-11-11T17:22:34.026431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519483363365855:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:34.026464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:34.029372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:34.029417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:34.029454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519483797834853:2048]
2025-11-11T17:22:34.029489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519483797834853:2048] [node 2] ICS09 handshake done sender: [1:7571519483797834850:2299] self: [1:7571519483797834851:0] peer: [2:7571519483363365851:0] socket: 18 qp: 3631
2025-11-11T17:22:34.029518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519483797834853:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:34.029548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:34.293965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519483797834853:2048] [node 2] ICS01 socket: 18 reason# Debug
2025-11-11T17:22:34.294016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519483797834853:2048] VirtualId# [1:7571519483797834851:0]
2025-11-11T17:22:34.294046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519483797834853:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:34.294103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:34.294156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:34.295163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:34.295409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519483363365855:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:34.295432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519483363365855:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:34.295506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:34.295541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519483363365855:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:34.303231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519483797834858:2302]) peer tries to resume nonexistent session Self# [2:7571519483363365851:0] Peer# [1:7571519483797834851:0]
2025-11-11T17:22:34.303308Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519483797834858:2302] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:34.310345Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519483363365857:2337] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:34.312832Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519483363365859:2338] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:34.312901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:34.312967Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:34.312993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519483363365855:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:34.313020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519483363365855:2048] VirtualId# [2:7571519483363365851:0]
2025-11-11T17:22:34.313081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:34.314724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:34.315579Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519483797834856:2301] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:34.333450Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519483797834859:2303] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:34.336852Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519483363365859:2338] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:34.337072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:34.337118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:34.337159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519483363365865:2048]
2025-11-11T17:22:34.337199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519483363365865:2048] [node 1] ICS09 handshake done sender: [2:7571519483363365859:2338] self: [2:7571519483363365862:0] peer: [1:7571519483797834855:0] socket: 20 qp: 3635
2025-11-11T17:22:34.337228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519483363365865:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:34.337274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:34.338546Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519483797834856:2301] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:34.338779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:34.338832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:34.338866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519483797834862:2048]
2025-11-11T17:22:34.338902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519483797834862:2048] [node 2] ICS09 handshake done sender: [1:7571519483797834856:2301] self: [1:7571519483797834855:0] peer: [2:7571519483363365862:0] socket: 18 qp: 3632
2025-11-11T17:22:34.338930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519483797834862:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:34.338957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:34.809862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:34.809934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [1 <-> 2] socket: Broken pipe
2025-11-11T17:22:34.809958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519483797834862:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:34.809982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519483797834862:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:34.810012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:34.811686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:34.812034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519483363365865:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:34.812057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519483363365865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:34.812084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:34.812121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519483363365865:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:34.814331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] socket: Connection reset by peer
2025-11-11T17:22:34.814624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519483797834862:2048] [node 2] ICS07 socket disconnect -1 reason# ECONNRESET
2025-11-11T17:22:34.814644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519483797834862:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:34.816480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519483797834862:2048] [node 2] ICS08 incoming handshake Self# [2:7571519483363365862:0] Peer# [1:7571519483797834855:0] Counter# 71 LastInputSerial# 2871
2025-11-11T17:22:34.819648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519483363365865:2048] [node 1] ICS08 incoming handshake Self# [1:7571519483797834855:0] Peer# [2:7571519483363365862:0] Counter# 2871 LastInputSerial# 71
2025-11-11T17:22:34.821837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519483797834864:2304] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:34.822239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:34.822299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519483797834862:2048] [node 2] ICS09 handshake done sender: [1:7571519483797834864:2304] self: [1:7571519483797834855:0] peer: [2:7571519483363365862:0] socket: 22 qp: 3641
2025-11-11T17:22:34.822700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519483797834862:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:34.822744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:34.824335Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519483363365869:2342] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:34.824693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:34.824747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519483363365865:2048] [node 1] ICS09 handshake done sender: [2:7571519483363365869:2342] self: [2:7571519483363365862:0] peer: [1:7571519483797834855:0] socket: 23 qp: 3640
2025-11-11T17:22:34.825234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519483363365865:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:34.825275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:35.254463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519483797834862:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-11T17:22:35.254513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519483797834862:2048] VirtualId# [1:7571519483797834855:0]
2025-11-11T17:22:35.254546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519483797834862:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:35.254593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:35.254638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:35.258648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:35.258882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519483363365865:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-11T17:22:35.258909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519483363365865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:35.258967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:35.259003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519483363365865:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:35.264181Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519487658333169:2345] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:35.266075Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:35.266106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519483363365865:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:35.266141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519483363365865:2048] VirtualId# [2:7571519483363365862:0]
2025-11-11T17:22:35.266197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:35.266233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:35.266831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519488092802168:2307]) peer tries to resume nonexistent session Self# [2:7571519483363365862:0] Peer# [1:7571519483797834855:0]
2025-11-11T17:22:35.266885Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519488092802168:2307] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
2025-11-11T17:22:35.268125Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519487658333168:2344] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:35.269027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [2:7571519487658333169:2345] held: no
2025-11-11T17:22:35.271670Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519487658333169:2345] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:35.268800Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519488092802166:2306] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:35.273661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:35.273694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:35.273725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519487658333173:2048]
2025-11-11T17:22:35.273756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519487658333173:2048] [node 1] ICS09 handshake done sender: [2:7571519487658333169:2345] self: [2:7571519487658333171:0] peer: [1:7571519488092802165:0] socket: 17 qp: 3643
2025-11-11T17:22:35.273779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519487658333173:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:35.273805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:35.272729Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519488092802166:2306] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:35.272943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:35.272982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:35.273022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519488092802170:2048]
2025-11-11T17:22:35.273058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519488092802170:2048] [node 2] ICS09 handshake done sender: [1:7571519488092802166:2306] self: [1:7571519488092802165:0] peer: [2:7571519487658333171:0] socket: 16 qp: 3642
2025-11-11T17:22:35.273089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519488092802170:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:35.273122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:35.474003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:35.474295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519488092802170:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:35.474320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519488092802170:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:35.474377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:35.474407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519488092802170:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:35.479598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:35.479846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519487658333173:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:35.479871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519487658333173:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:35.479956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:35.479993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519487658333173:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:35.486283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519487658333173:2048] [node 1] ICS08 incoming handshake Self# [1:7571519488092802165:0] Peer# [2:7571519487658333171:0] Counter# 555 LastInputSerial# 16
2025-11-11T17:22:35.490385Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519488092802172:2308] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:35.491420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:35.491480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519488092802170:2048] [node 2] ICS09 handshake done sender: [1:7571519488092802172:2308] self: [1:7571519488092802165:0] peer: [2:7571519487658333171:0] socket: 20 qp: 3645
2025-11-11T17:22:35.492030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519488092802170:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:35.492075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:35.497943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519488092802170:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:35.497982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519488092802170:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:35.498053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:35.498161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519488092802170:2048] [node 2] ICS08 incoming handshake Self# [2:7571519487658333171:0] Peer# [1:7571519488092802165:0] Counter# 16 LastInputSerial# 555
2025-11-11T17:22:35.498633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:35.498862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519488092802170:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:35.498894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519488092802170:2048] VirtualId# [1:7571519488092802165:0]
2025-11-11T17:22:35.498930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:35.511357Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519487658333176:2348] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:35.518112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:35.518180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519487658333173:2048] [node 1] ICS09 handshake done sender: [2:7571519487658333176:2348] self: [2:7571519487658333171:0] peer: [1:7571519488092802165:0] socket: 21 qp: 3646
2025-11-11T17:22:35.518823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519487658333173:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:35.518870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:35.519195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:35.519220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519487658333173:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:35.519247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519487658333173:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:35.519284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:35.519367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:35.519559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519487658333173:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:35.519575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519487658333173:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:35.533816Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:35.541714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519488092802178:2311]) peer tries to resume nonexistent session Self# [2:7571519487658333171:0] Peer# [1:7571519488092802165:0]
2025-11-11T17:22:35.541778Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519488092802178:2311] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:35.545852Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519487658333181:2350] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:35.550145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:35.550234Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:35.550259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519487658333173:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:35.550292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519487658333173:2048] VirtualId# [2:7571519487658333171:0]
2025-11-11T17:22:35.550349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:35.577866Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519488092802179:2312] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:35.578024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:35.586356Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519487658333184:2351] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:35.601889Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519488092802179:2312] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:35.602334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:35.602384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:35.602428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519488092802182:2048]
2025-11-11T17:22:35.602473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519488092802182:2048] [node 2] ICS09 handshake done sender: [1:7571519488092802179:2312] self: [1:7571519488092802180:0] peer: [2:7571519487658333183:0] socket: 17 qp: 3650
2025-11-11T17:22:35.602506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519488092802182:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:35.602539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:35.605747Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519487658333184:2351] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:35.606536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:35.606586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:35.606627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519487658333187:2048]
2025-11-11T17:22:35.606668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519487658333187:2048] [node 1] ICS09 handshake done sender: [2:7571519487658333184:2351] self: [2:7571519487658333183:0] peer: [1:7571519488092802180:0] socket: 16 qp: 3651
2025-11-11T17:22:35.606699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519487658333187:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:35.606730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:35.658578Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:35.658806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519488092802182:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-11T17:22:35.658835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519488092802182:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:35.658921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:35.658956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519488092802182:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:35.659857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:35.660058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519487658333187:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-11T17:22:35.660089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519487658333187:2048] VirtualId# [2:7571519487658333183:0]
2025-11-11T17:22:35.660115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519487658333187:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:35.660175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:35.660209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
Subscribe
2025-11-11T17:22:35.669487Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519487658333189:2352]) peer tries to resume nonexistent session Self# [1:7571519488092802180:0] Peer# [2:7571519487658333183:0]
2025-11-11T17:22:35.669682Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519487658333189:2352] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:35.671249Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519488092802184:2314] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:35.672226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:35.672302Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:35.672329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519488092802182:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:35.672359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519488092802182:2048] VirtualId# [1:7571519488092802180:0]
2025-11-11T17:22:35.672394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:35.693750Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519487658333192:2354] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:35.694020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:35.694311Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519488092802189:2316] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:35.694715Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519488092802187:2315] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:35.709943Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519487658333192:2354] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:35.713755Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519488092802187:2315] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:35.713917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:35.713998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:35.714059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519487658333196:2048]
2025-11-11T17:22:35.714109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519487658333196:2048] [node 1] ICS09 handshake done sender: [2:7571519487658333192:2354] self: [2:7571519487658333194:0] peer: [1:7571519488092802186:0] socket: 16 qp: 3653
2025-11-11T17:22:35.714149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519487658333196:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:35.714192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:35.718098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:35.718170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:35.718225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519488092802192:2048]
2025-11-11T17:22:35.718275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519488092802192:2048] [node 2] ICS09 handshake done sender: [1:7571519488092802187:2315] self: [1:7571519488092802186:0] peer: [2:7571519487658333194:0] socket: 17 qp: 3654
2025-11-11T17:22:35.718310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519488092802192:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:35.718347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:35.940363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:35.940582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519487658333196:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-11T17:22:35.940605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519487658333196:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:35.940659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:35.940688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519487658333196:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:35.943623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:35.943918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519488092802192:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:35.943944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519488092802192:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:35.944020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:35.944063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519488092802192:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:35.951251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519488092802192:2048] [node 2] ICS08 incoming handshake Self# [2:7571519487658333194:0] Peer# [1:7571519488092802186:0] Counter# 37 LastInputSerial# 1495
2025-11-11T17:22:35.952119Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519488092802194:2317] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:35.951592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519487658333196:2048] [node 1] ICS08 incoming handshake Self# [1:7571519488092802186:0] Peer# [2:7571519487658333194:0] Counter# 1495 LastInputSerial# 37
2025-11-11T17:22:35.952396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:35.952450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519488092802192:2048] [node 2] ICS09 handshake done sender: [1:7571519488092802194:2317] self: [1:7571519488092802186:0] peer: [2:7571519487658333194:0] socket: 22 qp: 3659
2025-11-11T17:22:35.953000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519488092802192:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:35.953041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:35.955293Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519487658333200:2357] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:35.955875Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:35.956948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:35.957000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519487658333196:2048] [node 1] ICS09 handshake done sender: [2:7571519487658333200:2357] self: [2:7571519487658333194:0] peer: [1:7571519488092802186:0] socket: 23 qp: 3660
2025-11-11T17:22:35.957631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519487658333196:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:35.957675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:35.957822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7571519350219379036:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:36.386496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:36.387202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:36.387472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519488092802192:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:36.387495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519488092802192:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:36.387522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:36.387559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519488092802192:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:36.388953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:36.389166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519487658333196:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-11T17:22:36.389189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519487658333196:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:36.389231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:36.389262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519487658333196:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:36.394082Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519492387769495:2319] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:36.394341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:36.394389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519488092802192:2048] [node 2] ICS09 handshake done sender: [1:7571519492387769495:2319] self: [1:7571519488092802186:0] peer: [2:7571519487658333194:0] socket: 16 qp: 3662
2025-11-11T17:22:36.394878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519488092802192:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:36.394927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:36.393617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519487658333196:2048] [node 1] ICS08 incoming handshake Self# [1:7571519488092802186:0] Peer# [2:7571519487658333194:0] Counter# 4858 LastInputSerial# 118
2025-11-11T17:22:36.398911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519488092802192:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:36.398946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519488092802192:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:36.398993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:36.399043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519488092802192:2048] [node 2] ICS08 incoming handshake Self# [2:7571519487658333194:0] Peer# [1:7571519488092802186:0] Counter# 118 LastInputSerial# 4858
2025-11-11T17:22:36.399286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:36.399513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519488092802192:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:36.400426Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519491953300500:2360] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:36.400728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:36.400773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519487658333196:2048] [node 1] ICS09 handshake done sender: [2:7571519491953300500:2360] self: [2:7571519487658333194:0] peer: [1:7571519488092802186:0] socket: 17 qp: 3661
2025-11-11T17:22:36.401564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519487658333196:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:36.401614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:36.401858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:36.402046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519487658333196:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-11T17:22:36.402076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519487658333196:2048] VirtualId# [2:7571519487658333194:0]
2025-11-11T17:22:36.402106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519487658333196:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:36.402142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:36.402177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:36.405701Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
TEvNodeDisconnected
2025-11-11T17:22:36.412149Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519492387769501:2322] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:36.412247Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:36.412271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519488092802192:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:36.412298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519488092802192:2048] VirtualId# [1:7571519488092802186:0]
2025-11-11T17:22:36.412353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:36.412386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:36.414579Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519491953300506:2362] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:36.424851Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519492387769501:2322] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:36.425134Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519491953300506:2362] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:36.425612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:36.425660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:36.425705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519491953300509:2048]
2025-11-11T17:22:36.425744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519491953300509:2048] [node 1] ICS09 handshake done sender: [2:7571519491953300506:2362] self: [2:7571519491953300505:0] peer: [1:7571519492387769502:0] socket: 17 qp: 3666
2025-11-11T17:22:36.425774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519491953300509:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:36.425808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:36.426442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:36.426483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:36.426521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519492387769504:2048]
2025-11-11T17:22:36.426554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519492387769504:2048] [node 2] ICS09 handshake done sender: [1:7571519492387769501:2322] self: [1:7571519492387769502:0] peer: [2:7571519491953300505:0] socket: 18 qp: 3665
2025-11-11T17:22:36.426581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519492387769504:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:36.426610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-11T17:22:36.759209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by debug command
2025-11-11T17:22:36.759670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519492387769504:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:36.759700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519492387769504:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:36.759746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:36.759779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519492387769504:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:36.761699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
TEvNodeDisconnected
2025-11-11T17:22:36.761941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519491953300509:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-11T17:22:36.761983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519491953300509:2048] VirtualId# [2:7571519491953300505:0]
2025-11-11T17:22:36.762035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519491953300509:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:36.762102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:36.762147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:36.775999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519491953300511:2363]) peer tries to resume nonexistent session Self# [1:7571519492387769502:0] Peer# [2:7571519491953300505:0]
2025-11-11T17:22:36.776065Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519491953300511:2363] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:36.776933Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519492387769506:2324] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:36.780483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:36.780566Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:36.780594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519492387769504:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:36.780626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519492387769504:2048] VirtualId# [1:7571519492387769502:0]
2025-11-11T17:22:36.780680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:36.783983Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519492387769508:2325] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:36.788458Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519491953300515:2365] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:36.788595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:36.789797Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519492387769510:2326] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:36.793476Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519491953300515:2365] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:36.793650Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519492387769510:2326] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:36.793898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:36.793977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:36.794019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519492387769514:2048]
2025-11-11T17:22:36.794056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519492387769514:2048] [node 2] ICS09 handshake done sender: [1:7571519492387769510:2326] self: [1:7571519492387769509:0] peer: [2:7571519491953300516:0] socket: 16 qp: 3672
2025-11-11T17:22:36.794086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519492387769514:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:36.794118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:36.795179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:36.795228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:36.795276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519491953300518:2048]
2025-11-11T17:22:36.795314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519491953300518:2048] [node 1] ICS09 handshake done sender: [2:7571519491953300515:2365] self: [2:7571519491953300516:0] peer: [1:7571519492387769509:0] socket: 18 qp: 3671
2025-11-11T17:22:36.795340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519491953300518:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:36.795367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-11T17:22:37.047297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by debug command
2025-11-11T17:22:37.047637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519491953300518:2048] [node 1] ICS07 socket disconnect 18 reason# Debug
2025-11-11T17:22:37.047664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519491953300518:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:37.047710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:37.047740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519491953300518:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:37.052643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:37.053144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519492387769514:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:37.053183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519492387769514:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:37.053275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:37.053317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519492387769514:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:37.056146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519492387769514:2048] [node 2] ICS08 incoming handshake Self# [2:7571519491953300516:0] Peer# [1:7571519492387769509:0] Counter# 40 LastInputSerial# 1622
2025-11-11T17:22:37.060703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519491953300518:2048] [node 1] ICS08 incoming handshake Self# [1:7571519492387769509:0] Peer# [2:7571519491953300516:0] Counter# 1622 LastInputSerial# 40
2025-11-11T17:22:37.061336Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519496682736812:2327] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:37.061628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:37.061685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519492387769514:2048] [node 2] ICS09 handshake done sender: [1:7571519496682736812:2327] self: [1:7571519492387769509:0] peer: [2:7571519491953300516:0] socket: 22 qp: 3675
2025-11-11T17:22:37.062282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519492387769514:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:37.062342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:37.066907Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:37.068277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519496248267818:2368] held: no
2025-11-11T17:22:37.068348Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519496248267818:2368] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:37.068815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:37.068866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519491953300518:2048] [node 1] ICS09 handshake done sender: [2:7571519496248267818:2368] self: [2:7571519491953300516:0] peer: [1:7571519492387769509:0] socket: 23 qp: 3676
2025-11-11T17:22:37.069410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519491953300518:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:37.069454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-11T17:22:37.577427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519492387769514:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-11T17:22:37.577482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519492387769514:2048] VirtualId# [1:7571519492387769509:0]
2025-11-11T17:22:37.577517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519492387769514:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:37.577579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:37.577648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:37.579781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:37.580030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519491953300518:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-11T17:22:37.580055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519491953300518:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:37.580087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:37.580144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519491953300518:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:37.583691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519496682736817:2329]) peer tries to resume nonexistent session Self# [2:7571519491953300516:0] Peer# [1:7571519492387769509:0]
2025-11-11T17:22:37.583751Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519496682736817:2329] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:37.584056Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519496248267821:2370] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:37.585002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:37.585071Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:37.585094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519491953300518:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:37.585125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519491953300518:2048] VirtualId# [2:7571519491953300516:0]
2025-11-11T17:22:37.585179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:37.596393Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519496682736818:2330] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:37.596551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:37.598241Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519496248267824:2371] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:37.612416Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519496682736818:2330] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:37.612731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:37.612778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:37.612823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519496682736821:2048]
2025-11-11T17:22:37.612862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519496682736821:2048] [node 2] ICS09 handshake done sender: [1:7571519496682736818:2330] self: [1:7571519496682736819:0] peer: [2:7571519496248267823:0] socket: 17 qp: 3678
2025-11-11T17:22:37.612896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519496682736821:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:37.612929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:37.617906Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519496248267824:2371] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:37.621916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:37.621965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:37.622012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519496248267827:2048]
2025-11-11T17:22:37.622052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519496248267827:2048] [node 1] ICS09 handshake done sender: [2:7571519496248267824:2371] self: [2:7571519496248267823:0] peer: [1:7571519496682736819:0] socket: 16 qp: 3679
2025-11-11T17:22:37.622084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519496248267827:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:37.622116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:38.089536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:38.089693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:38.089910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519496248267827:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:38.089934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519496248267827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:38.089959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:38.089992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519496248267827:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:38.092338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:38.092595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519496682736821:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:38.092617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519496682736821:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:38.092682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:38.092714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519496682736821:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:38.099340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519496248267827:2048] [node 1] ICS08 incoming handshake Self# [1:7571519496682736819:0] Peer# [2:7571519496248267823:0] Counter# 2332 LastInputSerial# 59
2025-11-11T17:22:38.099685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519496682736821:2048] [node 2] ICS08 incoming handshake Self# [2:7571519496248267823:0] Peer# [1:7571519496682736819:0] Counter# 59 LastInputSerial# 2332
2025-11-11T17:22:38.101581Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519500977704119:2332] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:38.101841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:38.101897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519496682736821:2048] [node 2] ICS09 handshake done sender: [1:7571519500977704119:2332] self: [1:7571519496682736819:0] peer: [2:7571519496248267823:0] socket: 22 qp: 3681
2025-11-11T17:22:38.102323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519496682736821:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:38.102370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:38.108004Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:38.109137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519500543235127:2373] held: no
2025-11-11T17:22:38.113668Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519500543235127:2373] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:38.115035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:38.115089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519496248267827:2048] [node 1] ICS09 handshake done sender: [2:7571519500543235127:2373] self: [2:7571519496248267823:0] peer: [1:7571519496682736819:0] socket: 23 qp: 3683
2025-11-11T17:22:38.115675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519496248267827:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:38.115719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:38.491627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:38.491839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:38.491880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519496248267827:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:38.491903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519496248267827:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:38.491927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:38.496048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:38.496295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519496248267827:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:38.496315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519496248267827:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:38.501730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] socket: Connection reset by peer
2025-11-11T17:22:38.502019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519496682736821:2048] [node 2] ICS07 socket disconnect 22 reason# ECONNRESET
2025-11-11T17:22:38.502046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519496682736821:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-11T17:22:38.502083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:38.502158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519496682736821:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:38.505300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519496682736821:2048] [node 2] ICS08 incoming handshake Self# [2:7571519496248267823:0] Peer# [1:7571519496682736819:0] Counter# 140 LastInputSerial# 5554
2025-11-11T17:22:38.518203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519496248267827:2048] [node 1] ICS08 incoming handshake Self# [1:7571519496682736819:0] Peer# [2:7571519496248267823:0] Counter# 5554 LastInputSerial# 140
2025-11-11T17:22:38.529874Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519500977704124:2334] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:38.530173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:38.530235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519496682736821:2048] [node 2] ICS09 handshake done sender: [1:7571519500977704124:2334] self: [1:7571519496682736819:0] peer: [2:7571519496248267823:0] socket: 19 qp: 3686
2025-11-11T17:22:38.530639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519496682736821:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:38.530686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:38.534504Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:38.537963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7571519500543235132:2376] held: no
2025-11-11T17:22:38.538040Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519500543235132:2376] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:38.538284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:38.538329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519496248267827:2048] [node 1] ICS09 handshake done sender: [2:7571519500543235132:2376] self: [2:7571519496248267823:0] peer: [1:7571519496682736819:0] socket: 20 qp: 3687
2025-11-11T17:22:38.538643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519496248267827:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:38.538684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-11T17:22:38.944811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7571519350219379036:2048] [node 1] ICP34 closed connection by debug command
2025-11-11T17:22:38.945843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:38.946079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519496248267827:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-11T17:22:38.946101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519496248267827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:38.946128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:38.946177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519496248267827:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:38.950349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:38.950791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519496682736821:2048] [node 2] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:38.950816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519496682736821:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:38.950837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:38.950877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519496682736821:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:38.953171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519496682736821:2048] [node 2] ICS08 incoming handshake Self# [2:7571519496248267823:0] Peer# [1:7571519496682736819:0] Counter# 206 LastInputSerial# 8222
2025-11-11T17:22:38.957486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519496248267827:2048] [node 1] ICS08 incoming handshake Self# [1:7571519496682736819:0] Peer# [2:7571519496248267823:0] Counter# 8222 LastInputSerial# 206
2025-11-11T17:22:38.959903Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519500977704129:2336] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:38.960252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:38.960319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519496682736821:2048] [node 2] ICS09 handshake done sender: [1:7571519500977704129:2336] self: [1:7571519496682736819:0] peer: [2:7571519496248267823:0] socket: 18 qp: 3691
2025-11-11T17:22:38.960948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519496682736821:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:38.960997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:38.962825Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519500543235137:2379] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:38.963223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:38.963301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519496248267827:2048] [node 1] ICS09 handshake done sender: [2:7571519500543235137:2379] self: [2:7571519496248267823:0] peer: [1:7571519496682736819:0] socket: 22 qp: 3690
2025-11-11T17:22:38.964074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519496248267827:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:38.964123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:39.338260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519496248267827:2048] [node 1] ICS01 socket: 22 reason# Debug
2025-11-11T17:22:39.338312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519496248267827:2048] VirtualId# [2:7571519496248267823:0]
2025-11-11T17:22:39.338347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519496248267827:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:39.338400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:39.338456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:39.339485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:39.339756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519496682736821:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-11T17:22:39.339780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519496682736821:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:39.339848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:39.339880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519496682736821:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:39.355167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519504838202436:2381]) peer tries to resume nonexistent session Self# [1:7571519496682736819:0] Peer# [2:7571519496248267823:0]
2025-11-11T17:22:39.355253Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519504838202436:2381] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:39.366622Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519505272671430:2338] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:39.369697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:39.369783Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:39.369810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519496682736821:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:39.369844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519496682736821:2048] VirtualId# [1:7571519496682736819:0]
2025-11-11T17:22:39.369898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:39.390121Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519505272671434:2340] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.394988Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519504838202440:2383] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.395118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:39.401978Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519505272671433:2339] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.409849Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519504838202440:2383] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:39.410157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:39.410212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:39.410258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519504838202443:2048]
2025-11-11T17:22:39.410298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519504838202443:2048] [node 1] ICS09 handshake done sender: [2:7571519504838202440:2383] self: [2:7571519504838202441:0] peer: [1:7571519505272671432:0] socket: 19 qp: 3695
2025-11-11T17:22:39.410330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519504838202443:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:39.410363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:39.411685Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519505272671433:2339] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:39.412354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:39.412414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:39.412452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519505272671438:2048]
2025-11-11T17:22:39.412487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519505272671438:2048] [node 2] ICS09 handshake done sender: [1:7571519505272671433:2339] self: [1:7571519505272671432:0] peer: [2:7571519504838202441:0] socket: 18 qp: 3696
2025-11-11T17:22:39.412516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519505272671438:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:39.412546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:39.489840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:39.490152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519504838202443:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-11T17:22:39.490178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519504838202443:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:39.490246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:39.490275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519504838202443:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:39.488518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:39.488732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:39.489051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [1 <-> 2] socket: Broken pipe
2025-11-11T17:22:39.489077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519505272671438:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:39.489102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519505272671438:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:39.489131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:39.489183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519505272671438:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:39.489219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519505272671438:2048] VirtualId# [1:7571519505272671432:0]
2025-11-11T17:22:39.489244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:39.495577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519505272671440:2341]) peer tries to resume nonexistent session Self# [2:7571519504838202441:0] Peer# [1:7571519505272671432:0]
2025-11-11T17:22:39.495656Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519505272671440:2341] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:39.496246Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519504838202445:2385] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:39.497020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:39.497093Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:39.497118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519504838202443:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:39.497145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519504838202443:2048] VirtualId# [2:7571519504838202441:0]
2025-11-11T17:22:39.497199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
2025-11-11T17:22:39.512716Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519505272671444:2343] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.513319Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519504838202450:2387] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.513421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:39.513807Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519505272671442:2342] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.521822Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519504838202450:2387] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:39.522081Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519505272671442:2342] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:39.522395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:39.522455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:39.522499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519505272671447:2048]
2025-11-11T17:22:39.522541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519505272671447:2048] [node 2] ICS09 handshake done sender: [1:7571519505272671442:2342] self: [1:7571519505272671441:0] peer: [2:7571519504838202451:0] socket: 17 qp: 3701
2025-11-11T17:22:39.522574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519505272671447:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:39.522606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:39.523898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:39.523944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:39.523981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519504838202453:2048]
2025-11-11T17:22:39.524018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519504838202453:2048] [node 1] ICS09 handshake done sender: [2:7571519504838202450:2387] self: [2:7571519504838202451:0] peer: [1:7571519505272671441:0] socket: 19 qp: 3700
2025-11-11T17:22:39.524046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519504838202453:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:39.524074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:39.589679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:39.589960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:39.590200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519504838202453:2048] [node 1] ICS01 socket: 19 reason# EndOfStream
2025-11-11T17:22:39.590233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519504838202453:2048] VirtualId# [2:7571519504838202451:0]
2025-11-11T17:22:39.590261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519504838202453:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:39.590328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:39.590358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:39.590741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:39.590936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519505272671447:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-11T17:22:39.590963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519505272671447:2048] VirtualId# [1:7571519505272671441:0]
2025-11-11T17:22:39.590988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519505272671447:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:39.591015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:39.591061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
Subscribe
2025-11-11T17:22:39.608960Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519505272671449:2344] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.609126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:39.609685Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519504838202456:2389] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.621858Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519505272671449:2344] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:39.622120Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519504838202456:2389] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-11T17:22:39.622400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:39.622448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:39.622491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519504838202459:2048]
2025-11-11T17:22:39.622528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519504838202459:2048] [node 1] ICS09 handshake done sender: [2:7571519504838202456:2389] self: [2:7571519504838202455:0] peer: [1:7571519505272671450:0] socket: 16 qp: 3703
2025-11-11T17:22:39.622560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519504838202459:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:39.622592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:39.625410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:39.625452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:39.625492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519505272671452:2048]
2025-11-11T17:22:39.625532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519505272671452:2048] [node 2] ICS09 handshake done sender: [1:7571519505272671449:2344] self: [1:7571519505272671450:0] peer: [2:7571519504838202455:0] socket: 17 qp: 3702
2025-11-11T17:22:39.625577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519505272671452:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:39.625626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:39.737873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:39.738037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:39.738303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519505272671452:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:39.738329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519505272671452:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:39.738361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:39.738400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519505272671452:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:39.739630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:39.739665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519504838202459:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:39.739686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519504838202459:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:39.739710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:39.740034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:39.740258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519504838202459:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:39.740276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519504838202459:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:39.742173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519504838202459:2048] [node 1] ICS08 incoming handshake Self# [1:7571519505272671450:0] Peer# [2:7571519504838202455:0] Counter# 343 LastInputSerial# 10
2025-11-11T17:22:39.744322Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519505272671454:2346] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:39.749869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:39.749934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519505272671452:2048] [node 2] ICS09 handshake done sender: [1:7571519505272671454:2346] self: [1:7571519505272671450:0] peer: [2:7571519504838202455:0] socket: 20 qp: 3704
2025-11-11T17:22:39.750659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519505272671452:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:39.750711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:39.756705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519505272671452:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:39.756741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519505272671452:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-11T17:22:39.756808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:39.756850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519505272671452:2048] [node 2] ICS08 incoming handshake Self# [2:7571519504838202455:0] Peer# [1:7571519505272671450:0] Counter# 10 LastInputSerial# 343
2025-11-11T17:22:39.757094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:39.757314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519505272671452:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-11T17:22:39.757343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519505272671452:2048] VirtualId# [1:7571519505272671450:0]
2025-11-11T17:22:39.757374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:39.758959Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519504838202462:2391] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:39.766063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:39.766127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519504838202459:2048] [node 1] ICS09 handshake done sender: [2:7571519504838202462:2391] self: [2:7571519504838202455:0] peer: [1:7571519505272671450:0] socket: 21 qp: 3705
2025-11-11T17:22:39.766834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519504838202459:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:39.766882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:39.767211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [2 <-> 1] socket: Broken pipe
2025-11-11T17:22:39.767235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [2:7571519504838202459:2048] [node 1] ICS13 reestablish connection
2025-11-11T17:22:39.767277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519504838202459:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:39.767301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:39.767380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:39.767585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519504838202459:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:39.767602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519504838202459:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:39.770938Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [1 <-> 0] network-related error occured on handshake: incoming handshake Peer# <unknown>(::1) Socket error# connection unexpectedly closed state# ReceiveRequest processed# 0 remain# 52
2025-11-11T17:22:39.775836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7571519350653848058:2048] [node 2] ICP12 (actor [1:7571519505272671460:2349]) peer tries to resume nonexistent session Self# [2:7571519504838202455:0] Peer# [1:7571519505272671450:0]
2025-11-11T17:22:39.775889Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [1:7571519505272671460:2349] [node 2] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:39.776881Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [2:7571519504838202467:2393] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake
2025-11-11T17:22:39.777840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:16840) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:39.777918Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7571519350219379036:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:39.777942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519504838202459:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:39.777972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519504838202459:2048] VirtualId# [2:7571519504838202455:0]
2025-11-11T17:22:39.778088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 50073
TEvUndelivered Cookie# 50074
TEvUndelivered Cookie# 50075
TEvUndelivered Cookie# 50076
TEvUndelivered Cookie# 50077
TEvUndelivered Cookie# 50078
2025-11-11T17:22:39.792546Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519505272671461:2350] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.792672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519350653848058:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:39.798054Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519504838202470:2394] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:39.809837Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519505272671461:2350] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:39.810157Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519504838202470:2394] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:39.810389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519350219379036:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:39.810431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:39.810475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519504838202473:2048]
2025-11-11T17:22:39.810512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519504838202473:2048] [node 1] ICS09 handshake done sender: [2:7571519504838202470:2394] self: [2:7571519504838202469:0] peer: [1:7571519505272671462:0] socket: 16 qp: 3710
2025-11-11T17:22:39.810545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519504838202473:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:39.810577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:39.811064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519350653848058:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:39.811102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:39.811135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519505272671464:2048]
2025-11-11T17:22:39.811167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519505272671464:2048] [node 2] ICS09 handshake done sender: [1:7571519505272671461:2350] self: [1:7571519505272671462:0] peer: [2:7571519504838202469:0] socket: 17 qp: 3709
2025-11-11T17:22:39.811190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519505272671464:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:39.811217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-11T17:22:40.309759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7571519350653848058:2048] [node 2] ICP34 closed connection by debug command
2025-11-11T17:22:40.309872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:749: [1 <-> 2] socket: Broken pipe
2025-11-11T17:22:40.309896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:516: Session [1:7571519505272671464:2048] [node 2] ICS13 reestablish connection
2025-11-11T17:22:40.309920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519505272671464:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-11T17:22:40.309946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:40.311154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:40.311397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519504838202473:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-11T17:22:40.311421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519504838202473:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:40.311482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:40.311515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519504838202473:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:40.312127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:40.312368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519505272671464:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-11T17:22:40.312386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519505272671464:2048] [node 2] ICS15 start handshake
2025-11-11T17:22:40.317626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [1:7571519505272671464:2048] [node 2] ICS08 incoming handshake Self# [2:7571519504838202469:0] Peer# [1:7571519505272671462:0] Counter# 84 LastInputSerial# 3440
2025-11-11T17:22:40.319141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:249: Session [2:7571519504838202473:2048] [node 1] ICS08 incoming handshake Self# [1:7571519505272671462:0] Peer# [2:7571519504838202469:0] Counter# 3440 LastInputSerial# 84
2025-11-11T17:22:40.321077Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519509567638762:2352] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:40.321342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:40.321396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519505272671464:2048] [node 2] ICS09 handshake done sender: [1:7571519509567638762:2352] self: [1:7571519505272671462:0] peer: [2:7571519504838202469:0] socket: 22 qp: 3714
2025-11-11T17:22:40.322036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519505272671464:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:40.322106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:40.323927Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519509133169773:2396] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:40.324276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:40.324329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519504838202473:2048] [node 1] ICS09 handshake done sender: [2:7571519509133169773:2396] self: [2:7571519504838202469:0] peer: [1:7571519505272671462:0] socket: 23 qp: 3713
2025-11-11T17:22:40.324883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519504838202473:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:40.324924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-11T17:22:40.899419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:40.899692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [1:7571519505272671464:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-11T17:22:40.899716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519505272671464:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:40.899768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:40.899798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [1:7571519505272671464:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-11T17:22:40.894740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519504838202473:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-11T17:22:40.894790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519504838202473:2048] VirtualId# [2:7571519504838202469:0]
2025-11-11T17:22:40.894821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519504838202473:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-11T17:22:40.894869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:40.894907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:40.903550Z :INTERCONNECT INFO: interconnect_handshake.cpp:1588: Handshake [1:7571519509567638767:2354] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake
2025-11-11T17:22:40.904481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7571519350653848058:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:18613) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-11T17:22:40.904543Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7571519350653848058:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-11T17:22:40.904565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519505272671464:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-11T17:22:40.904593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519350653848058:2048] [node 2] ICP30 unregister session Session# [1:7571519505272671464:2048] VirtualId# [1:7571519505272671462:0]
2025-11-11T17:22:40.904640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
2025-11-11T17:22:40.902864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7571519350219379036:2048] [node 1] ICP12 (actor [2:7571519509133169776:2398]) peer tries to resume nonexistent session Self# [1:7571519505272671462:0] Peer# [2:7571519504838202469:0]
2025-11-11T17:22:40.902925Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1114: Handshake [2:7571519509133169776:2398] [node 1] ICH08 Continuation request rejected by proxy
2025-11-11T17:22:40.914182Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519509133169780:2400] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:40.913775Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519509567638769:2355] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:40.914572Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519509567638771:2356] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:40.914280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519350219379036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:40.917755Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519509133169780:2400] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:40.919405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519350219379036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:22:40.917897Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519509567638771:2356] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:40.919437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:40.918264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519350653848058:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:22:40.919468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519350219379036:2048] [node 1] ICP22 created new session: [2:7571519509133169783:2048]
2025-11-11T17:22:40.918311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:40.919498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519509133169783:2048] [node 1] ICS09 handshake done sender: [2:7571519509133169780:2400] self: [2:7571519509133169781:0] peer: [1:7571519509567638770:0] socket: 17 qp: 3719
2025-11-11T17:22:40.918350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519350653848058:2048] [node 2] ICP22 created new session: [1:7571519509567638775:2048]
2025-11-11T17:22:40.919520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519509133169783:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:40.918387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519509567638775:2048] [node 2] ICS09 handshake done sender: [1:7571519509567638771:2356] self: [1:7571519509567638770:0] peer: [2:7571519509133169781:0] socket: 16 qp: 3718
2025-11-11T17:22:40.918415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519509567638775:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:40.918442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:40.919543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Subscribe
TEvNodeConnected
2025-11-11T17:22:41.245679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:41.245924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519509133169783:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-11T17:22:41.245946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519509133169783:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:41.246015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:41.246050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519509133169783:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:41.259068Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52
2025-11-11T17:22:41.260217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:16840) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-11T17:22:41.260257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519509133169783:2048] [node 1] ICS15 start handshake
2025-11-11T17:22:41.260814Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# ::1(::1:16840) Couldn't connect to any resolved address
2025-11-11T17:22:41.261014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7571519350219379036:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:16840) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-11T17:22:41.261057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519509133169783:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-11T17:22:41.261096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519350219379036:2048] [node 1] ICP30 unregister session Session# [2:7571519509133169783:2048] VirtualId# [2:7571519509133169781:0]
2025-11-11T17:22:41.261155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:41.261185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7571519350219379036:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:16840) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-11T17:22:40.893352Z
2025-11-11T17:22:41.261200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:16840) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-11T17:22:40.893352Z
TEvNodeDisconnected
2025-11-11T17:22:41.273736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7571519350219379036:2048] [node 1] ICP33 wake up from error state
2025-11-11T17:22:41.357991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519513137992203:2048] [node 2] ICP01 ready to work
2025-11-11T17:22:41.375590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519513205637654:2048] [node 1] ICP01 ready to work
Subscribe
2025-11-11T17:22:41.395909Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519513137992212:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:41.397423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519513137992203:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:41.400283Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519513205637664:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:41.415248Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519513137992212:2056] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:41.415611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519513137992203:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:41.415661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:41.415711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519513137992203:2048] [node 2] ICP22 created new session: [1:7571519513137992215:2048]
2025-11-11T17:22:41.415748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519513137992215:2048] [node 2] ICS09 handshake done sender: [1:7571519513137992212:2056] self: [1:7571519513137992213:0] peer: [2:7571519513205637663:0] socket: 17 qp: 3721
2025-11-11T17:22:41.415780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519513137992215:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:41.415816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:41.417744Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519513205637664:2056] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-11T17:22:41.418113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519513205637654:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:41.418157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:41.418200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519513205637654:2048] [node 1] ICP22 created new session: [2:7571519513205637667:2048]
2025-11-11T17:22:41.418243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519513205637667:2048] [node 1] ICS09 handshake done sender: [2:7571519513205637664:2056] self: [2:7571519513205637663:0] peer: [1:7571519513137992213:0] socket: 16 qp: 3720
2025-11-11T17:22:41.418279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519513205637667:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:41.418305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:41.478379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [1:7571519513137992203:2048] [node 2] ICP31 proxy http called
2025-11-11T17:22:41.479568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [2:7571519513205637654:2048] [node 1] ICP31 proxy http called
TEvNodeDisconnected
2025-11-11T17:22:41.501084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:41.501303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519513205637667:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-11T17:22:41.501343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519513205637654:2048] [node 1] ICP30 unregister session Session# [2:7571519513205637667:2048] VirtualId# [2:7571519513205637663:0]
2025-11-11T17:22:41.501372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519513205637667:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:41.501444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:41.501493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:41.569965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519511678915844:2048] [node 2] ICP01 ready to work
2025-11-11T17:22:41.581420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519513592156898:2048] [node 1] ICP01 ready to work
2025-11-11T17:22:41.726178Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519511678915853:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:41.727774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519511678915844:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:41.734087Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519513592156908:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:41.743726Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519511678915853:2056] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:41.744082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519511678915844:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:41.744134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:41.744177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519511678915844:2048] [node 2] ICP22 created new session: [1:7571519511678915856:2048]
2025-11-11T17:22:41.744221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519511678915856:2048] [node 2] ICS09 handshake done sender: [1:7571519511678915853:2056] self: [1:7571519511678915854:0] peer: [2:7571519513592156907:0] socket: 17 qp: 3722
2025-11-11T17:22:41.744249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519511678915856:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:41.744289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:41.745799Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519513592156908:2056] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:41.748134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519513592156898:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:41.748203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:41.748270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519513592156898:2048] [node 1] ICP22 created new session: [2:7571519513592156911:2048]
2025-11-11T17:22:41.748323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519513592156911:2048] [node 1] ICS09 handshake done sender: [2:7571519513592156908:2056] self: [2:7571519513592156907:0] peer: [1:7571519511678915854:0] socket: 16 qp: 3723
2025-11-11T17:22:41.748361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519513592156911:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:41.748390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
RECEIVED TEvTest
2025-11-11T17:22:43.347585Z :INTERCONNECT CRIT: interconnect_tcp_session.cpp:498: Session [2:7571519513592156911:2048] [node 1] ICS31 serialized event Type# 0x7ff00000 is too large
2025-11-11T17:22:43.347631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519513592156911:2048] [node 1] ICS01 socket: 16 reason# EventTooLarge
2025-11-11T17:22:43.347674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519513592156898:2048] [node 1] ICP30 unregister session Session# [2:7571519513592156911:2048] VirtualId# [2:7571519513592156907:0]
2025-11-11T17:22:43.347710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519513592156911:2048] [node 1] ICS25 shutdown socket, reason# EventTooLarge
2025-11-11T17:22:43.347765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:43.351500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [1 <-> 2] connection closed by peer
2025-11-11T17:22:43.351853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7571519511678915856:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-11T17:22:43.351883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7571519511678915844:2048] [node 2] ICP30 unregister session Session# [1:7571519511678915856:2048] VirtualId# [1:7571519511678915854:0]
2025-11-11T17:22:43.351911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7571519511678915856:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:43.351939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [2] disconnected
2025-11-11T17:22:43.351981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [2] session destroyed
TEvUndelivered
2025-11-11T17:22:43.515192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
2025-11-11T17:22:43.521118Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519520268850450:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:43.521275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7571519511678915844:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:22:43.521717Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519522182091506:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:22:43.525386Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519520268850450:2058] [node 2] ICH04 handshake succeeded
2025-11-11T17:22:43.525764Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519522182091506:2057] [node 1] ICH04 handshake succeeded
2025-11-11T17:22:43.526282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7571519513592156898:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-11T17:22:43.526354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:22:43.526407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519513592156898:2048] [node 1] ICP22 created new session: [2:7571519522182091509:2048]
2025-11-11T17:22:43.526456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519522182091509:2048] [node 1] ICS09 handshake done sender: [2:7571519522182091506:2057] self: [2:7571519522182091505:0] peer: [1:7571519520268850451:0] socket: 16 qp: 3725
2025-11-11T17:22:43.526495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519522182091509:2048] [node 1] ICS10 traffic start
2025-11-11T17:22:43.526538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
2025-11-11T17:22:43.528139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7571519511678915844:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-11T17:22:43.528203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:22:43.528252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519511678915844:2048] [node 2] ICP22 created new session: [1:7571519520268850453:2048]
RECEIVED TEvTest
2025-11-11T17:22:43.528297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519520268850453:2048] [node 2] ICS09 handshake done sender: [1:7571519520268850450:2058] self: [1:7571519520268850451:0] peer: [2:7571519522182091505:0] socket: 17 qp: 3724
2025-11-11T17:22:43.528338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519520268850453:2048] [node 2] ICS10 traffic start
2025-11-11T17:22:43.528375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:22:43.548726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:22:43.548947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7571519522182091509:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-11T17:22:43.548980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7571519513592156898:2048] [node 1] ICP30 unregister session Session# [2:7571519522182091509:2048] VirtualId# [2:7571519522182091505:0]
2025-11-11T17:22:43.549011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519522182091509:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:22:43.549072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:22:43.549105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:129: [1] session destroyed
written 2560000 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 8351744 bytes
written 2721720 bytes
written 4133621 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4196015 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
2025-11-11T17:23:09.667034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7571519632024089858:2048] [node 2] ICP01 ready to work
2025-11-11T17:23:09.682137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7571519632709416158:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:23:09.689177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519632709416167:2056] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42363
2025-11-11T17:23:09.697932Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [2:7571519632709416167:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:23:09.699035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7571519632709416158:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-11T17:23:09.699306Z :INTERCONNECT WARN: interconnect_handshake.cpp:543: Handshake [1:7571519632024089868:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-11T17:23:09.699336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:957: Handshake [1:7571519632024089868:2056] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-11T17:23:09.700249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1392: Handshake [2:7571519632709416169:2057] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42363
2025-11-11T17:23:09.700412Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [1:7571519632024089868:2056] [node 2] ICH04 handshake succeeded
2025-11-11T17:23:09.700854Z :INTERCONNECT INFO: interconnect_handshake.cpp:415: Handshake [2:7571519632709416167:2056] [node 1] ICH04 handshake succeeded
2025-11-11T17:23:09.701055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7571519632024089858:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-11T17:23:09.701116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-11T17:23:09.701147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7571519632024089858:2048] [node 2] ICP22 created new session: [1:7571519632024089871:2048]
2025-11-11T17:23:09.701173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [1:7571519632024089871:2048] [node 2] ICS09 handshake done sender: [1:7571519632024089868:2056] self: [1:7571519632024089867:0] peer: [2:7571519632709416168:0] socket: 24 qp: -1
2025-11-11T17:23:09.701208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [1:7571519632024089871:2048] [node 2] ICS10 traffic start
2025-11-11T17:23:09.701247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [2] connected
2025-11-11T17:23:09.702331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7571519632709416158:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-11T17:23:09.702412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-11T17:23:09.702459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7571519632709416158:2048] [node 1] ICP22 created new session: [2:7571519632709416170:2048]
2025-11-11T17:23:09.702511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:263: Session [2:7571519632709416170:2048] [node 1] ICS09 handshake done sender: [2:7571519632709416167:2056] self: [2:7571519632709416168:0] peer: [1:7571519632024089867:0] socket: 25 qp: -1
2025-11-11T17:23:09.702575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:288: Session [2:7571519632709416170:2048] [node 1] ICS10 traffic start
2025-11-11T17:23:09.702678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:314: [1] connected
Updated MaxRTT# 0.024803s
Updated MaxRTT# 0.026687s
Updated MaxRTT# 0.026953s
Updated MaxRTT# 0.026954s
Updated MaxRTT# 0.028684s
Updated MaxRTT# 0.057750s
Updated MaxRTT# 0.057756s
Updated MaxRTT# 0.058326s
Updated MaxRTT# 0.060111s
2025-11-11T17:23:19.700241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:715: [2 <-> 1] connection closed by peer
2025-11-11T17:23:19.700487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:533: Session [2:7571519632709416170:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-11T17:23:19.700516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7571519632709416170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-11T17:23:19.700610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:562: [1] disconnected
2025-11-11T17:23:19.700658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:505: Session [2:7571519632709416170:2048] [node 1] ICS15 start handshake
2025-11-11T17:23:19.707854Z :INTERCONNECT_NETWORK NOTICE: interconnect_handshake.cpp:1581: [2 <-> 1] network-related error occured on handshake: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3467) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52
